kd-engine 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. kd_engine-1.0.0.dist-info/METADATA +19 -0
  2. kd_engine-1.0.0.dist-info/RECORD +118 -0
  3. kd_engine-1.0.0.dist-info/WHEEL +5 -0
  4. kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
  5. kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
  6. kd_engine-1.0.0.dist-info/top_level.txt +1 -0
  7. ytk/__init__.py +2 -0
  8. ytk/__main__.py +21 -0
  9. ytk/batch/__init__.py +39 -0
  10. ytk/batch/runner.py +1149 -0
  11. ytk/cache/__init__.py +34 -0
  12. ytk/cache/v2.py +562 -0
  13. ytk/cleaning/density.py +40 -0
  14. ytk/cleaning/fillers.py +18 -0
  15. ytk/cleaning/sentence_boundary.py +30 -0
  16. ytk/cleaning/service.py +62 -0
  17. ytk/cli.py +1281 -0
  18. ytk/commands/__init__.py +3 -0
  19. ytk/commands/merge.py +600 -0
  20. ytk/config.py +757 -0
  21. ytk/dedup/service.py +166 -0
  22. ytk/doctor.py +891 -0
  23. ytk/dotenv.py +334 -0
  24. ytk/errors.py +38 -0
  25. ytk/extraction/service.py +451 -0
  26. ytk/extraction/strategies/base.py +16 -0
  27. ytk/extraction/strategies/code_block.py +285 -0
  28. ytk/extraction/strategies/heuristic.py +791 -0
  29. ytk/extraction/strategies/local_model.py +56 -0
  30. ytk/extraction/text_context.py +177 -0
  31. ytk/extraction/visual_context.py +159 -0
  32. ytk/ingestion/adapters/__init__.py +1 -0
  33. ytk/ingestion/adapters/_common.py +865 -0
  34. ytk/ingestion/adapters/base.py +160 -0
  35. ytk/ingestion/adapters/bilibili.py +271 -0
  36. ytk/ingestion/adapters/douyin.py +445 -0
  37. ytk/ingestion/adapters/tiktok.py +102 -0
  38. ytk/ingestion/adapters/xiaohongshu.py +758 -0
  39. ytk/ingestion/adapters/youtube.py +74 -0
  40. ytk/ingestion/browser_acquired.py +220 -0
  41. ytk/ingestion/http_client.py +43 -0
  42. ytk/ingestion/local_import.py +202 -0
  43. ytk/ingestion/media.py +468 -0
  44. ytk/ingestion/providers/asr.py +902 -0
  45. ytk/ingestion/providers/base.py +13 -0
  46. ytk/ingestion/providers/dtk_client.py +816 -0
  47. ytk/ingestion/providers/minimax.py +350 -0
  48. ytk/ingestion/providers/transcript_api.py +336 -0
  49. ytk/ingestion/providers/xhs_client.py +646 -0
  50. ytk/ingestion/providers/ytdlp.py +400 -0
  51. ytk/ingestion/registry.py +189 -0
  52. ytk/ingestion/resolver.py +417 -0
  53. ytk/ingestion/search.py +56 -0
  54. ytk/ingestion/service.py +167 -0
  55. ytk/ingestion/subtitles.py +602 -0
  56. ytk/ingestion/transcript_cache.py +212 -0
  57. ytk/ingestion/url_parser.py +53 -0
  58. ytk/ingestion/video_frames.py +292 -0
  59. ytk/ingestion/ytdlp_acquire.py +754 -0
  60. ytk/logging.py +383 -0
  61. ytk/models/common.py +20 -0
  62. ytk/models/content.py +451 -0
  63. ytk/models/knowledge.py +43 -0
  64. ytk/models/migrations.py +262 -0
  65. ytk/models/skill.py +13 -0
  66. ytk/models/source.py +265 -0
  67. ytk/models/topic.py +18 -0
  68. ytk/models/transcript.py +30 -0
  69. ytk/ocr/__init__.py +65 -0
  70. ytk/ocr/engine.py +245 -0
  71. ytk/ocr/layout.py +347 -0
  72. ytk/ocr/pipeline.py +440 -0
  73. ytk/ocr/rapidocr_engine.py +144 -0
  74. ytk/pipeline/acquisition_checkpoint.py +240 -0
  75. ytk/pipeline/artifacts.py +18 -0
  76. ytk/pipeline/orchestrator.py +598 -0
  77. ytk/pipeline/processing_identity.py +18 -0
  78. ytk/pipeline/retry.py +728 -0
  79. ytk/pipeline/runners.py +578 -0
  80. ytk/pipeline/state.py +1107 -0
  81. ytk/pipeline/v2.py +1863 -0
  82. ytk/py.typed +1 -0
  83. ytk/quality/__init__.py +9 -0
  84. ytk/quality/completeness.py +260 -0
  85. ytk/renderers/base.py +9 -0
  86. ytk/renderers/citation_alignment.py +617 -0
  87. ytk/renderers/evidence_formatter.py +362 -0
  88. ytk/renderers/json_renderer.py +13 -0
  89. ytk/renderers/jsonl_renderer.py +18 -0
  90. ytk/renderers/knowledge_doc_renderer.py +1984 -0
  91. ytk/renderers/markdown_renderer.py +540 -0
  92. ytk/renderers/obsidian_renderer.py +138 -0
  93. ytk/renderers/skill_package.py +21 -0
  94. ytk/renderers/source_links.py +309 -0
  95. ytk/renderers/summary_renderer.py +163 -0
  96. ytk/renderers/v2.py +171 -0
  97. ytk/renderers/v2_jsonl.py +165 -0
  98. ytk/renderers/v2_markdown.py +477 -0
  99. ytk/renderers/v2_obsidian.py +113 -0
  100. ytk/renderers/v2_summary.py +220 -0
  101. ytk/segmentation/change_points.py +50 -0
  102. ytk/segmentation/embeddings.py +45 -0
  103. ytk/segmentation/service.py +318 -0
  104. ytk/skill_builder/__init__.py +1 -0
  105. ytk/skill_builder/builder.py +419 -0
  106. ytk/skill_builder/templates/SKILL.md.j2 +85 -0
  107. ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
  108. ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
  109. ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
  110. ytk/skill_builder/templates/references/commands.md.j2 +34 -0
  111. ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
  112. ytk/skill_builder/templates/references/configs.md.j2 +31 -0
  113. ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
  114. ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
  115. ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
  116. ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
  117. ytk/skill_builder/templates.py +29 -0
  118. ytk/text_normalization.py +44 -0
@@ -0,0 +1,3 @@
1
+ from ytk.commands.merge import deduplicate_atoms, load_knowledge, merge_knowledge_directories
2
+
3
+ __all__ = ["deduplicate_atoms", "load_knowledge", "merge_knowledge_directories"]
ytk/commands/merge.py ADDED
@@ -0,0 +1,600 @@
1
+ """T41.1 — cross-source merge for v2 knowledge bundles.
2
+
3
+ The merge order is fixed by the plan (§5.5 T41.1): **validate each input
4
+ schema → merge sources, contents and the evidence registry → merge knowledge
5
+ atoms last**. Everything the merged artifact cites is carried with it:
6
+
7
+ * evidence is deduplicated by ``(source_id, source_revision, evidence_id)``;
8
+ the same evidence id reused by another source/revision is namespaced and the
9
+ atom references are rewritten, never silently overwritten ("同一局部 ID /
10
+ 同来源不同修订不能互相覆盖");
11
+ * a genuinely conflicting entry (same identity, different payload) is a hard
12
+ :class:`MergeConflictError`;
13
+ * atoms merge only when type *and* normalized content *and* the negation /
14
+ condition signature all match — opposite conclusions stay separate;
15
+ * confidence is never raised because more sources repeat a claim;
16
+ * referenced local assets are copied into ``<output_dir>/assets/<source>/`` and
17
+ re-pointed with package-relative paths, so the merged product still resolves
18
+ after the original input directories are moved.
19
+
20
+ Legacy v1 artifacts keep working:
21
+
22
+ * a *minimal* v1 ``knowledge.json`` (``video_id`` + ``knowledge_atoms`` only)
23
+ merges through the historical dictionary path, byte-compatible with the old
24
+ CLI output;
25
+ * a full v1 ``ExtractionBundle`` is migrated with
26
+ :func:`ytk.models.migrations.migrate_extraction_bundle_v1` and merges as v2.
27
+
28
+ Anything else (bad JSON, unsupported ``schema_version``, non-extraction JSON,
29
+ missing/conflicting evidence, unreachable assets) is rejected with a clear
30
+ :class:`MergeInputError` instead of producing a half-resolvable artifact.
31
+ """
32
+
33
+ from __future__ import annotations
34
+
35
+ import hashlib
36
+ import json
37
+ import re
38
+ import shutil
39
+ from collections.abc import Iterable, Mapping, Sequence
40
+ from dataclasses import dataclass, field
41
+ from pathlib import Path
42
+ from typing import Any, Literal
43
+
44
+ from pydantic import ValidationError
45
+
46
+ from ytk.dedup.service import DeduplicationService
47
+ from ytk.errors import UserFacingError
48
+ from ytk.models.content import (
49
+ ContentBundle,
50
+ EvidenceRefV2,
51
+ ExtractionBundleV2,
52
+ KnowledgeAtomV2,
53
+ MediaAsset,
54
+ TopicGroup,
55
+ locator_key,
56
+ )
57
+ from ytk.models.knowledge import ExtractionBundle
58
+ from ytk.models.migrations import migrate_extraction_bundle_v1
59
+ from ytk.models.source import ContentRef
60
+
61
+ InputKind = Literal["v2", "v1", "legacy", "unknown"]
62
+
63
+ _V1_REQUIRED_KEYS = {"metadata", "transcript_source", "topic_segments", "skill_spec"}
64
+ _SAFE_SLUG_RE = re.compile(r"[^A-Za-z0-9._-]+")
65
+
66
+
67
+ class MergeInputError(UserFacingError):
68
+ """An input artifact cannot be merged (bad schema, missing or fake evidence)."""
69
+
70
+
71
+ class MergeConflictError(MergeInputError):
72
+ """Two inputs make incompatible claims about the same identity."""
73
+
74
+
75
+ @dataclass
76
+ class _Artifact:
77
+ path: Path
78
+ root: Path
79
+ bundle: ExtractionBundleV2 | None = None
80
+ payload: dict[str, Any] = field(default_factory=dict)
81
+ kind: InputKind = "unknown"
82
+
83
+
84
+ # ---------------------------------------------------------------------------
85
+ # Loading / schema validation
86
+ # ---------------------------------------------------------------------------
87
+
88
+
89
+ def load_knowledge(path: Path) -> dict[str, Any]:
90
+ """Parse one JSON artifact; raise :class:`MergeInputError` when unreadable."""
91
+ path = Path(path)
92
+ try:
93
+ payload = json.loads(path.read_text(encoding="utf-8"))
94
+ except OSError as exc:
95
+ raise MergeInputError(f"cannot read merge input {path}: {exc}") from exc
96
+ except (json.JSONDecodeError, UnicodeError) as exc:
97
+ raise MergeInputError(f"merge input {path} is not valid JSON: {exc}") from exc
98
+ if not isinstance(payload, dict):
99
+ raise MergeInputError(f"merge input {path} is not a JSON object")
100
+ return payload
101
+
102
+
103
+ def classify_payload(path: Path, payload: Mapping[str, Any]) -> InputKind:
104
+ """Classify a parsed artifact as v2 / full v1 / legacy minimal / unknown."""
105
+ if "schema_version" in payload:
106
+ version = payload["schema_version"]
107
+ if version != 2:
108
+ raise MergeInputError(
109
+ f"merge input {path} declares unsupported schema_version={version!r}"
110
+ )
111
+ return "v2"
112
+ if "knowledge_atoms" in payload and "video_id" in payload:
113
+ if _V1_REQUIRED_KEYS & set(payload):
114
+ return "v1"
115
+ return "legacy"
116
+ return "unknown"
117
+
118
+
119
+ def _format_validation_error(exc: ValidationError) -> str:
120
+ return "; ".join(
121
+ f"{'.'.join(str(part) for part in error.get('loc', ())) or '<root>'}: "
122
+ f"{error.get('msg', 'invalid value')}"
123
+ for error in exc.errors()[:5]
124
+ )
125
+
126
+
127
+ def load_bundle(path: Path, payload: Mapping[str, Any] | None = None) -> ExtractionBundleV2:
128
+ """Validate one v1/v2 artifact and return a v2 bundle (strict, never guesses)."""
129
+ path = Path(path)
130
+ data = dict(payload) if payload is not None else load_knowledge(path)
131
+ kind = classify_payload(path, data)
132
+ if kind == "legacy":
133
+ raise MergeInputError(
134
+ f"merge input {path} is a legacy minimal knowledge.json and cannot be "
135
+ "combined with extraction bundles"
136
+ )
137
+ if kind == "unknown":
138
+ raise MergeInputError(
139
+ f"merge input {path} is not a YTK knowledge extraction artifact"
140
+ )
141
+ if kind == "v2":
142
+ try:
143
+ return ExtractionBundleV2.model_validate(data)
144
+ except ValidationError as exc:
145
+ raise MergeInputError(
146
+ f"merge input {path} is not a valid v2 bundle: {_format_validation_error(exc)}"
147
+ ) from exc
148
+ try:
149
+ v1 = ExtractionBundle.model_validate(data)
150
+ except ValidationError as exc:
151
+ raise MergeInputError(
152
+ f"merge input {path} is not a valid v1 knowledge bundle: "
153
+ f"{_format_validation_error(exc)}"
154
+ ) from exc
155
+ return migrate_extraction_bundle_v1(v1)
156
+
157
+
158
+ # ---------------------------------------------------------------------------
159
+ # Assets
160
+ # ---------------------------------------------------------------------------
161
+
162
+
163
+ def _slug(value: str) -> str:
164
+ slug = _SAFE_SLUG_RE.sub("-", value).strip("-")
165
+ return slug or "source"
166
+
167
+
168
+ def _asset_candidates(asset: MediaAsset, artifacts: Sequence[_Artifact], content_key: str) -> list[Path]:
169
+ candidates: list[Path] = []
170
+ for artifact in artifacts:
171
+ if artifact.bundle is None:
172
+ continue
173
+ if content_key not in {content.content_key for content in artifact.bundle.contents}:
174
+ continue
175
+ for candidate in (
176
+ artifact.path.parent / asset.relative_path,
177
+ artifact.root / asset.relative_path,
178
+ ):
179
+ if candidate.resolve().is_relative_to(artifact.root.resolve()):
180
+ candidates.append(candidate)
181
+ seen: set[Path] = set()
182
+ unique: list[Path] = []
183
+ for candidate in candidates:
184
+ if candidate not in seen:
185
+ seen.add(candidate)
186
+ unique.append(candidate)
187
+ return unique
188
+
189
+
190
+ def _copy_assets(
191
+ bundle: ExtractionBundleV2,
192
+ artifacts: Sequence[_Artifact],
193
+ output_dir: Path,
194
+ ) -> list[ContentBundle]:
195
+ """Copy every referenced asset into the package and rewrite relative paths."""
196
+ relocated: list[ContentBundle] = []
197
+ for content in bundle.contents:
198
+ new_assets: list[MediaAsset] = []
199
+ for asset in content.assets:
200
+ candidates = _asset_candidates(asset, artifacts, content.content_key)
201
+ available = [path for path in candidates if path.is_file()]
202
+ source_path = next((path for path in available if (
203
+ path.stat().st_size == asset.byte_size
204
+ and hashlib.sha256(path.read_bytes()).hexdigest() == asset.sha256
205
+ )), None)
206
+ if source_path is None:
207
+ if available:
208
+ raise MergeInputError(
209
+ f"asset {asset.asset_id!r} of {content.content_key} has a mismatching digest or size"
210
+ )
211
+ raise MergeInputError(
212
+ f"asset {asset.asset_id!r} of {asset.source_id} was not found "
213
+ f"(expected at {asset.relative_path}); the merged output would not "
214
+ "resolve after moving the input directories"
215
+ )
216
+ target_dir = output_dir / "assets" / _slug(asset.source_id)
217
+ if not target_dir.resolve().is_relative_to(output_dir.resolve()):
218
+ raise MergeInputError("asset destination escapes the merged output package")
219
+ target_dir.mkdir(parents=True, exist_ok=True)
220
+ target = target_dir / source_path.name
221
+ if target.is_symlink():
222
+ raise MergeInputError("asset destination must not be a symbolic link")
223
+ if target.exists() and hashlib.sha256(target.read_bytes()).hexdigest() != asset.sha256:
224
+ target = target.with_name(f"{target.stem}-{asset.sha256[:20]}{target.suffix}")
225
+ if target.exists():
226
+ if target.read_bytes() != source_path.read_bytes():
227
+ raise MergeConflictError(
228
+ f"two different assets map to {target} for source {asset.source_id}"
229
+ )
230
+ else:
231
+ shutil.copyfile(source_path, target)
232
+ relative = target.relative_to(output_dir).as_posix()
233
+ new_assets.append(asset.model_copy(update={"relative_path": relative}))
234
+ relocated.append(content.model_copy(update={"assets": new_assets}))
235
+ return relocated
236
+
237
+
238
+ # ---------------------------------------------------------------------------
239
+ # Core merge
240
+ # ---------------------------------------------------------------------------
241
+
242
+
243
+ def _merge_sources(
244
+ collected: dict[str, ContentRef], incoming_sequence: Sequence[ContentRef]
245
+ ) -> dict[str, ContentRef]:
246
+ for ref in incoming_sequence:
247
+ current = collected.get(ref.source_id)
248
+ if current is None:
249
+ collected[ref.source_id] = ref
250
+ continue
251
+ if current.platform != ref.platform or current.content_type != ref.content_type:
252
+ raise MergeConflictError(
253
+ f"source {ref.source_id!r} is declared with conflicting platform/content "
254
+ f"type ({current.platform}/{current.content_type} vs "
255
+ f"{ref.platform}/{ref.content_type})"
256
+ )
257
+ aliases = list(current.alias_urls)
258
+ for alias in ref.alias_urls:
259
+ if alias not in aliases:
260
+ aliases.append(alias)
261
+ display = {**dict(ref.display), **dict(current.display)}
262
+ collected[ref.source_id] = current.model_copy(
263
+ update={"alias_urls": aliases, "display": display}
264
+ )
265
+ return collected
266
+
267
+
268
+ def _same_content(left: ContentBundle, right: ContentBundle) -> bool:
269
+ left_blocks = {block.block_id: block.model_dump(mode="json") for block in left.blocks}
270
+ right_blocks = {block.block_id: block.model_dump(mode="json") for block in right.blocks}
271
+ return (
272
+ left_blocks == right_blocks
273
+ and [asset.model_dump(exclude={"relative_path"}) for asset in left.assets]
274
+ == [asset.model_dump(exclude={"relative_path"}) for asset in right.assets]
275
+ and left.completeness.status == right.completeness.status
276
+ and left.source.source_id == right.source.source_id
277
+ )
278
+
279
+
280
+ def _claim_evidence_id(
281
+ evidence: EvidenceRefV2, used: dict[str, tuple[str, str, str]]
282
+ ) -> str:
283
+ """Return the merged id for ``evidence`` (namespace on a real collision)."""
284
+ identity = (evidence.source_id, evidence.source_revision, evidence.evidence_id)
285
+ current = used.get(evidence.evidence_id)
286
+ if current is None or current == identity:
287
+ used[evidence.evidence_id] = identity
288
+ return evidence.evidence_id
289
+ candidate = f"{evidence.source_id}@{evidence.source_revision}#{evidence.evidence_id}"
290
+ counter = 2
291
+ while candidate in used and used[candidate] != identity:
292
+ candidate = (
293
+ f"{evidence.source_id}@{evidence.source_revision}#{evidence.evidence_id}~{counter}"
294
+ )
295
+ counter += 1
296
+ used[candidate] = identity
297
+ return candidate
298
+
299
+
300
+ def _evidence_fingerprint(evidence: EvidenceRefV2) -> tuple[str, ...]:
301
+ return (
302
+ evidence.source_id,
303
+ evidence.source_revision,
304
+ evidence.block_id,
305
+ evidence.quote,
306
+ locator_key(evidence.locator),
307
+ evidence.processor,
308
+ )
309
+
310
+
311
+ def _unique_topic_id(topic: TopicGroup, used_topic_ids: set[str]) -> str:
312
+ if topic.topic_id not in used_topic_ids:
313
+ used_topic_ids.add(topic.topic_id)
314
+ return topic.topic_id
315
+ counter = 2
316
+ candidate = f"{topic.topic_id}~{counter}"
317
+ while candidate in used_topic_ids:
318
+ counter += 1
319
+ candidate = f"{topic.topic_id}~{counter}"
320
+ used_topic_ids.add(candidate)
321
+ return candidate
322
+
323
+
324
+ def merge_bundles(
325
+ bundles: Sequence[ExtractionBundleV2],
326
+ *,
327
+ artifacts: Sequence[_Artifact] = (),
328
+ output_dir: Path | None = None,
329
+ ) -> ExtractionBundleV2:
330
+ """Merge already-validated v2 bundles into one self-contained bundle.
331
+
332
+ ``output_dir`` (when given) receives the copied assets; every asset path in
333
+ the result is relative to it. The function is deterministic for a given
334
+ input order — inputs are expected in sorted artifact order.
335
+ """
336
+ source_map: dict[str, ContentRef] = {}
337
+ content_map: dict[tuple[str, str], ContentBundle] = {}
338
+ evidence_index: dict[str, EvidenceRefV2] = {}
339
+ evidence_used: dict[str, tuple[str, str, str]] = {}
340
+ evidence_fingerprints: dict[str, tuple[str, ...]] = {}
341
+ topics: list[TopicGroup] = []
342
+ topic_ids: set[str] = set()
343
+ atoms: list[KnowledgeAtomV2] = []
344
+
345
+ for bundle in bundles:
346
+ _merge_sources(source_map, bundle.sources)
347
+ for content in bundle.contents:
348
+ key = (content.source.source_id, content.source_revision)
349
+ existing = content_map.get(key)
350
+ if existing is None:
351
+ content_map[key] = content
352
+ elif not _same_content(existing, content):
353
+ raise MergeConflictError(
354
+ f"conflicting content for revision {content.content_key}: two inputs "
355
+ "carry different blocks for the same source revision"
356
+ )
357
+ evidence_mapping: dict[str, str] = {}
358
+ for evidence_id, evidence in bundle.evidence_index.items():
359
+ new_id = _claim_evidence_id(evidence, evidence_used)
360
+ fingerprint = _evidence_fingerprint(evidence)
361
+ previous = evidence_fingerprints.get(new_id)
362
+ if previous is not None and previous != fingerprint:
363
+ raise MergeConflictError(
364
+ f"evidence {new_id!r} is registered twice with different content "
365
+ "(conflicting evidence id)"
366
+ )
367
+ evidence_mapping[evidence_id] = new_id
368
+ evidence_index[new_id] = evidence.model_copy(update={"evidence_id": new_id})
369
+ evidence_fingerprints[new_id] = fingerprint
370
+ for topic in bundle.topics:
371
+ new_topic_id = _unique_topic_id(topic, topic_ids)
372
+ topics.append(
373
+ topic.model_copy(update={"topic_id": new_topic_id})
374
+ if new_topic_id != topic.topic_id
375
+ else topic
376
+ )
377
+ for atom in bundle.knowledge:
378
+ mapped_ids: list[str] = []
379
+ for evidence_id in atom.evidence_ids:
380
+ if evidence_id not in evidence_mapping:
381
+ raise MergeInputError(
382
+ f"atom {atom.atom_id!r} references evidence {evidence_id!r} that is "
383
+ "missing from its bundle's evidence_index"
384
+ )
385
+ mapped_ids.append(evidence_mapping[evidence_id])
386
+ atoms.append(atom.model_copy(update={"evidence_ids": mapped_ids}))
387
+
388
+ if output_dir is not None:
389
+ assembled = ExtractionBundleV2.model_construct(
390
+ sources=list(source_map.values()),
391
+ contents=list(content_map.values()),
392
+ evidence_index=evidence_index,
393
+ topics=topics,
394
+ knowledge=atoms,
395
+ metadata={},
396
+ skill_spec=None,
397
+ )
398
+ relocated = _copy_assets(assembled, artifacts, Path(output_dir))
399
+ content_map = {
400
+ (content.source.source_id, content.source_revision): content
401
+ for content in relocated
402
+ }
403
+
404
+ deduped = DeduplicationService().merge_v2(atoms)
405
+ merged_metadata: dict[str, Any] = {
406
+ "title": f"Merged knowledge from {len(source_map)} sources",
407
+ "source_ids": [ref.source_id for ref in source_map.values()],
408
+ "merged_schema": "v2",
409
+ }
410
+ try:
411
+ return ExtractionBundleV2(
412
+ sources=list(source_map.values()),
413
+ contents=list(content_map.values()),
414
+ evidence_index=evidence_index,
415
+ topics=topics,
416
+ knowledge=deduped,
417
+ metadata=merged_metadata,
418
+ skill_spec=None,
419
+ )
420
+ except ValidationError as exc:
421
+ raise MergeConflictError(
422
+ "merged bundle failed reference validation: " + _format_validation_error(exc)
423
+ ) from exc
424
+
425
+
426
+ def merge_bundle_files(
427
+ paths: Sequence[Path],
428
+ *,
429
+ artifact_root: Path | None = None,
430
+ output_dir: Path | None = None,
431
+ ) -> ExtractionBundleV2:
432
+ """Validate and merge the given v1/v2 artifacts (no legacy minimal inputs)."""
433
+ artifacts: list[_Artifact] = []
434
+ bundles: list[ExtractionBundleV2] = []
435
+ for path in paths:
436
+ path = Path(path)
437
+ payload = load_knowledge(path)
438
+ kind = classify_payload(path, payload)
439
+ if kind == "legacy":
440
+ raise MergeInputError(
441
+ f"merge input {path} is a legacy minimal knowledge.json and cannot be "
442
+ "combined with extraction bundles"
443
+ )
444
+ bundle = load_bundle(path, payload)
445
+ artifacts.append(
446
+ _Artifact(
447
+ path=path,
448
+ root=Path(artifact_root) if artifact_root else path.parent,
449
+ bundle=bundle,
450
+ payload=payload,
451
+ kind=kind,
452
+ )
453
+ )
454
+ bundles.append(bundle)
455
+ return merge_bundles(bundles, artifacts=artifacts, output_dir=output_dir)
456
+
457
+
458
+ def merge_bundle_directories(input_dir: Path, output_dir: Path) -> ExtractionBundleV2:
459
+ """Merge every ``knowledge.json`` under ``input_dir`` into ``output_dir``."""
460
+ input_dir = Path(input_dir)
461
+ paths = sorted(input_dir.rglob("knowledge.json"))
462
+ if not paths:
463
+ raise MergeInputError(f"no knowledge.json found under {input_dir}")
464
+ return merge_bundle_files(paths, artifact_root=input_dir, output_dir=output_dir)
465
+
466
+
467
+ # ---------------------------------------------------------------------------
468
+ # Legacy (minimal v1) merge path — byte-compatible with the old CLI output
469
+ # ---------------------------------------------------------------------------
470
+
471
+
472
+ def _merge_legacy_artifacts(
473
+ artifacts: Sequence[tuple[Path, dict[str, Any]]], output_path: Path
474
+ ) -> dict[str, Any]:
475
+ all_atoms: list[dict[str, Any]] = []
476
+ source_videos: list[str] = []
477
+ for path, data in artifacts:
478
+ video_id = data.get("video_id", path.parent.name)
479
+ source_videos.append(video_id)
480
+ for atom in data.get("knowledge_atoms", []):
481
+ if not isinstance(atom, dict):
482
+ raise MergeInputError(f"merge input {path} has a non-object knowledge atom")
483
+ atom = dict(atom)
484
+ atom["video_id"] = video_id
485
+ all_atoms.append(atom)
486
+
487
+ merged_atoms = deduplicate_atoms(all_atoms)
488
+ result: dict[str, Any] = {
489
+ "title": f"Merged knowledge from {len(source_videos)} videos",
490
+ "source_count": len(source_videos),
491
+ "source_videos": source_videos,
492
+ "atoms": merged_atoms,
493
+ }
494
+ output_path = Path(output_path)
495
+ output_path.parent.mkdir(parents=True, exist_ok=True)
496
+ output_path.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8")
497
+ return result
498
+
499
+
500
+ def _write_bundle(bundle: ExtractionBundleV2, output_path: Path) -> None:
501
+ output_path = Path(output_path)
502
+ output_path.parent.mkdir(parents=True, exist_ok=True)
503
+ output_path.write_text(
504
+ json.dumps(bundle.model_dump(mode="json"), indent=2, ensure_ascii=False) + "\n",
505
+ encoding="utf-8",
506
+ )
507
+
508
+
509
+ def merge_knowledge_directories(input_dir: Path, output_path: Path) -> dict[str, Any]:
510
+ """Merge every ``knowledge.json`` under ``input_dir`` into ``output_path``.
511
+
512
+ Dispatches on the input schema: all-legacy minimal v1 inputs keep the
513
+ historical dictionary output; any full v1/v2 artifact produces a v2
514
+ :class:`ExtractionBundleV2` JSON with self-contained relative assets.
515
+ """
516
+ input_dir = Path(input_dir)
517
+ output_path = Path(output_path)
518
+ paths = sorted(input_dir.rglob("knowledge.json"))
519
+ if not paths:
520
+ raise MergeInputError(f"no knowledge.json found under {input_dir}")
521
+
522
+ payloads: list[tuple[Path, dict[str, Any], InputKind]] = []
523
+ for path in paths:
524
+ payload = load_knowledge(path)
525
+ payloads.append((path, payload, classify_payload(path, payload)))
526
+
527
+ if all(kind == "legacy" for _path, _payload, kind in payloads):
528
+ return _merge_legacy_artifacts(
529
+ [(path, payload) for path, payload, _kind in payloads], output_path
530
+ )
531
+
532
+ legacy_paths = [str(path) for path, _payload, kind in payloads if kind == "legacy"]
533
+ if legacy_paths:
534
+ raise MergeInputError(
535
+ "cannot mix legacy minimal knowledge.json with extraction bundles: "
536
+ + ", ".join(legacy_paths)
537
+ )
538
+ unknown_paths = [str(path) for path, _payload, kind in payloads if kind == "unknown"]
539
+ if unknown_paths:
540
+ raise MergeInputError(
541
+ "these merge inputs are not YTK knowledge artifacts: " + ", ".join(unknown_paths)
542
+ )
543
+
544
+ bundle = merge_bundle_files(paths, artifact_root=input_dir, output_dir=output_path.parent)
545
+ _write_bundle(bundle, output_path)
546
+ return {
547
+ "title": f"Merged knowledge from {len(bundle.sources)} sources",
548
+ "source_count": len(bundle.sources),
549
+ "sources": [ref.public_dict() for ref in bundle.sources],
550
+ "schema_version": 2,
551
+ "atoms": [atom.model_dump(mode="json") for atom in bundle.knowledge],
552
+ }
553
+
554
+
555
+ # ---------------------------------------------------------------------------
556
+ # Legacy dictionary-level helpers (public API kept for existing callers)
557
+ # ---------------------------------------------------------------------------
558
+
559
+
560
+ def deduplicate_atoms(atoms: list[dict]) -> list[dict]:
561
+ seen: dict[tuple[str, str], dict] = {}
562
+ for atom in atoms:
563
+ key = (atom.get("atom_type", ""), atom.get("content", "").casefold())
564
+ if key in seen:
565
+ existing = seen[key]
566
+ sources = existing.setdefault("source_videos", [])
567
+ vid = atom.get("video_id", "")
568
+ if vid and vid not in sources:
569
+ sources.append(vid)
570
+ if existing.get("video_id") and existing["video_id"] not in sources:
571
+ sources.insert(0, existing["video_id"])
572
+ if atom.get("confidence", 0) > existing.get("confidence", 0):
573
+ existing["confidence"] = atom["confidence"]
574
+ else:
575
+ atom_copy = dict(atom)
576
+ vid = atom_copy.pop("video_id", "")
577
+ if vid:
578
+ atom_copy["source_videos"] = [vid]
579
+ seen[key] = atom_copy
580
+ return list(seen.values())
581
+
582
+
583
+ def merge_atom_refs(atoms: Iterable[KnowledgeAtomV2]) -> list[KnowledgeAtomV2]:
584
+ """Merge v2 atoms without a bundle (thin wrapper over the dedup service)."""
585
+ return DeduplicationService().merge_v2(list(atoms))
586
+
587
+
588
+ __all__ = [
589
+ "MergeConflictError",
590
+ "MergeInputError",
591
+ "classify_payload",
592
+ "deduplicate_atoms",
593
+ "load_bundle",
594
+ "load_knowledge",
595
+ "merge_atom_refs",
596
+ "merge_bundle_directories",
597
+ "merge_bundle_files",
598
+ "merge_bundles",
599
+ "merge_knowledge_directories",
600
+ ]