kd-engine 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. kd_engine-1.0.0.dist-info/METADATA +19 -0
  2. kd_engine-1.0.0.dist-info/RECORD +118 -0
  3. kd_engine-1.0.0.dist-info/WHEEL +5 -0
  4. kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
  5. kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
  6. kd_engine-1.0.0.dist-info/top_level.txt +1 -0
  7. ytk/__init__.py +2 -0
  8. ytk/__main__.py +21 -0
  9. ytk/batch/__init__.py +39 -0
  10. ytk/batch/runner.py +1149 -0
  11. ytk/cache/__init__.py +34 -0
  12. ytk/cache/v2.py +562 -0
  13. ytk/cleaning/density.py +40 -0
  14. ytk/cleaning/fillers.py +18 -0
  15. ytk/cleaning/sentence_boundary.py +30 -0
  16. ytk/cleaning/service.py +62 -0
  17. ytk/cli.py +1281 -0
  18. ytk/commands/__init__.py +3 -0
  19. ytk/commands/merge.py +600 -0
  20. ytk/config.py +757 -0
  21. ytk/dedup/service.py +166 -0
  22. ytk/doctor.py +891 -0
  23. ytk/dotenv.py +334 -0
  24. ytk/errors.py +38 -0
  25. ytk/extraction/service.py +451 -0
  26. ytk/extraction/strategies/base.py +16 -0
  27. ytk/extraction/strategies/code_block.py +285 -0
  28. ytk/extraction/strategies/heuristic.py +791 -0
  29. ytk/extraction/strategies/local_model.py +56 -0
  30. ytk/extraction/text_context.py +177 -0
  31. ytk/extraction/visual_context.py +159 -0
  32. ytk/ingestion/adapters/__init__.py +1 -0
  33. ytk/ingestion/adapters/_common.py +865 -0
  34. ytk/ingestion/adapters/base.py +160 -0
  35. ytk/ingestion/adapters/bilibili.py +271 -0
  36. ytk/ingestion/adapters/douyin.py +445 -0
  37. ytk/ingestion/adapters/tiktok.py +102 -0
  38. ytk/ingestion/adapters/xiaohongshu.py +758 -0
  39. ytk/ingestion/adapters/youtube.py +74 -0
  40. ytk/ingestion/browser_acquired.py +220 -0
  41. ytk/ingestion/http_client.py +43 -0
  42. ytk/ingestion/local_import.py +202 -0
  43. ytk/ingestion/media.py +468 -0
  44. ytk/ingestion/providers/asr.py +902 -0
  45. ytk/ingestion/providers/base.py +13 -0
  46. ytk/ingestion/providers/dtk_client.py +816 -0
  47. ytk/ingestion/providers/minimax.py +350 -0
  48. ytk/ingestion/providers/transcript_api.py +336 -0
  49. ytk/ingestion/providers/xhs_client.py +646 -0
  50. ytk/ingestion/providers/ytdlp.py +400 -0
  51. ytk/ingestion/registry.py +189 -0
  52. ytk/ingestion/resolver.py +417 -0
  53. ytk/ingestion/search.py +56 -0
  54. ytk/ingestion/service.py +167 -0
  55. ytk/ingestion/subtitles.py +602 -0
  56. ytk/ingestion/transcript_cache.py +212 -0
  57. ytk/ingestion/url_parser.py +53 -0
  58. ytk/ingestion/video_frames.py +292 -0
  59. ytk/ingestion/ytdlp_acquire.py +754 -0
  60. ytk/logging.py +383 -0
  61. ytk/models/common.py +20 -0
  62. ytk/models/content.py +451 -0
  63. ytk/models/knowledge.py +43 -0
  64. ytk/models/migrations.py +262 -0
  65. ytk/models/skill.py +13 -0
  66. ytk/models/source.py +265 -0
  67. ytk/models/topic.py +18 -0
  68. ytk/models/transcript.py +30 -0
  69. ytk/ocr/__init__.py +65 -0
  70. ytk/ocr/engine.py +245 -0
  71. ytk/ocr/layout.py +347 -0
  72. ytk/ocr/pipeline.py +440 -0
  73. ytk/ocr/rapidocr_engine.py +144 -0
  74. ytk/pipeline/acquisition_checkpoint.py +240 -0
  75. ytk/pipeline/artifacts.py +18 -0
  76. ytk/pipeline/orchestrator.py +598 -0
  77. ytk/pipeline/processing_identity.py +18 -0
  78. ytk/pipeline/retry.py +728 -0
  79. ytk/pipeline/runners.py +578 -0
  80. ytk/pipeline/state.py +1107 -0
  81. ytk/pipeline/v2.py +1863 -0
  82. ytk/py.typed +1 -0
  83. ytk/quality/__init__.py +9 -0
  84. ytk/quality/completeness.py +260 -0
  85. ytk/renderers/base.py +9 -0
  86. ytk/renderers/citation_alignment.py +617 -0
  87. ytk/renderers/evidence_formatter.py +362 -0
  88. ytk/renderers/json_renderer.py +13 -0
  89. ytk/renderers/jsonl_renderer.py +18 -0
  90. ytk/renderers/knowledge_doc_renderer.py +1984 -0
  91. ytk/renderers/markdown_renderer.py +540 -0
  92. ytk/renderers/obsidian_renderer.py +138 -0
  93. ytk/renderers/skill_package.py +21 -0
  94. ytk/renderers/source_links.py +309 -0
  95. ytk/renderers/summary_renderer.py +163 -0
  96. ytk/renderers/v2.py +171 -0
  97. ytk/renderers/v2_jsonl.py +165 -0
  98. ytk/renderers/v2_markdown.py +477 -0
  99. ytk/renderers/v2_obsidian.py +113 -0
  100. ytk/renderers/v2_summary.py +220 -0
  101. ytk/segmentation/change_points.py +50 -0
  102. ytk/segmentation/embeddings.py +45 -0
  103. ytk/segmentation/service.py +318 -0
  104. ytk/skill_builder/__init__.py +1 -0
  105. ytk/skill_builder/builder.py +419 -0
  106. ytk/skill_builder/templates/SKILL.md.j2 +85 -0
  107. ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
  108. ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
  109. ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
  110. ytk/skill_builder/templates/references/commands.md.j2 +34 -0
  111. ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
  112. ytk/skill_builder/templates/references/configs.md.j2 +31 -0
  113. ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
  114. ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
  115. ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
  116. ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
  117. ytk/skill_builder/templates.py +29 -0
  118. ytk/text_normalization.py +44 -0
@@ -0,0 +1,362 @@
1
+ """T33.1 — the single evidence formatting / validation authority for v2.
2
+
3
+ Everything a renderer writes about evidence (link, readable locator, quoted
4
+ excerpt) is produced here; nothing downstream formats ``evidence.start`` /
5
+ ``evidence.end`` by hand any more. The same module is the *validator*: a
6
+ citation that does not resolve to a real block of the matching source
7
+ revision, claims an image that does not exist, carries a bounding box outside
8
+ the image, cites time beyond the media duration, or invents a timestamp for an
9
+ image note is rejected with a clear error instead of being rendered (A22).
10
+
11
+ Design rules (plan §4.3, contract §1.6):
12
+
13
+ * image notes never get ``[00:00]``-style citations;
14
+ * ``locator`` on an evidence ref must equal the locator of the block it points
15
+ at — a mismatching coordinate is a fabrication, not "close enough";
16
+ * the quoted excerpt must be present in the block's raw (or cleaned) text;
17
+ whitespace differences are tolerated, re-wording is not;
18
+ * validation is deterministic and offline (no network, no model).
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import re
24
+ from collections.abc import Iterable, Sequence
25
+ from typing import Any
26
+
27
+ from ytk.errors import UserFacingError
28
+ from ytk.models.content import (
29
+ ContentBundle,
30
+ EvidenceRefV2,
31
+ ExtractionBundleV2,
32
+ ImageRegionLocator,
33
+ ParagraphLocator,
34
+ TimeRangeLocator,
35
+ VideoFrameLocator,
36
+ locator_key,
37
+ )
38
+ from ytk.models.source import ContentRef
39
+ from ytk.renderers.source_links import (
40
+ is_image_note,
41
+ platform_link,
42
+ readable_locator,
43
+ )
44
+
45
+ Locator = TimeRangeLocator | ParagraphLocator | ImageRegionLocator | VideoFrameLocator
46
+
47
+ #: Time citations past the media duration are rejected beyond this slack.
48
+ DURATION_TOLERANCE_SECONDS = 1.0
49
+
50
+ _WHITESPACE_RE = re.compile(r"\s+")
51
+
52
+
53
+ class EvidenceValidationError(UserFacingError):
54
+ """An evidence reference is fabricated or cannot resolve to its source."""
55
+
56
+
57
+ def _normalize_text(text: str) -> str:
58
+ return _WHITESPACE_RE.sub(" ", str(text)).strip()
59
+
60
+
61
+ def find_source(bundle: ExtractionBundleV2, source_id: str) -> ContentRef:
62
+ """Return the registered :class:`ContentRef` for ``source_id``."""
63
+ for ref in bundle.sources:
64
+ if ref.source_id == source_id:
65
+ return ref
66
+ raise EvidenceValidationError(
67
+ f"evidence references source {source_id!r} that is not registered in the bundle"
68
+ )
69
+
70
+
71
+ def find_content(bundle: ExtractionBundleV2, source_id: str, source_revision: str) -> ContentBundle:
72
+ """Return the content bundle for ``(source_id, source_revision)``."""
73
+ for content in bundle.contents:
74
+ if content.source.source_id == source_id and content.source_revision == source_revision:
75
+ return content
76
+ raise EvidenceValidationError(
77
+ f"evidence references revision {source_id}@{source_revision} that is not present in contents"
78
+ )
79
+
80
+
81
+ def content_ref_for(content: ContentBundle) -> ContentRef:
82
+ return content.source
83
+
84
+
85
+ def _positive_number(value: Any) -> float | None:
86
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
87
+ return None
88
+ if value <= 0:
89
+ return None
90
+ return float(value)
91
+
92
+
93
+ def known_duration(content: ContentBundle, bundle: ExtractionBundleV2 | None = None) -> float | None:
94
+ """Media duration in seconds when the artifacts record it.
95
+
96
+ Checked (in order): ``content.metadata["duration"]``,
97
+ ``content.metadata["duration_seconds"]``, the same keys on the bundle
98
+ metadata, then the largest timestamp carried by a time locator. Unknown
99
+ duration means "cannot check" — never a licence to accept any value.
100
+ """
101
+ for payload in (content.metadata, (bundle.metadata if bundle is not None else {})):
102
+ for key in ("duration", "duration_seconds", "media_duration"):
103
+ value = _positive_number(payload.get(key)) if isinstance(payload, dict) else None
104
+ if value is not None:
105
+ return value
106
+ latest: float | None = None
107
+ for block in content.blocks:
108
+ locator = block.locator
109
+ if isinstance(locator, TimeRangeLocator):
110
+ latest = locator.time.end if latest is None else max(latest, locator.time.end)
111
+ elif isinstance(locator, VideoFrameLocator):
112
+ latest = locator.time if latest is None else max(latest, locator.time)
113
+ return latest
114
+
115
+
116
+ def image_dimensions(content: ContentBundle, image_index: int) -> tuple[int, int] | None:
117
+ """Declared ``(width, height)`` for ``image_index`` when the artifacts say.
118
+
119
+ Accepts ``metadata["image_sizes"]`` as a list of ``[w, h]`` pairs or of
120
+ ``{"width": w, "height": h}`` dicts (1-based index order). Returns
121
+ ``None`` when the artifact does not record dimensions.
122
+ """
123
+ sizes = content.metadata.get("image_sizes") if isinstance(content.metadata, dict) else None
124
+ if not isinstance(sizes, Sequence) or not sizes:
125
+ return None
126
+ if image_index < 1 or image_index > len(sizes):
127
+ return None
128
+ entry = sizes[image_index - 1]
129
+ if isinstance(entry, dict):
130
+ width = _positive_number(entry.get("width"))
131
+ height = _positive_number(entry.get("height"))
132
+ elif isinstance(entry, Sequence) and len(entry) == 2:
133
+ width = _positive_number(entry[0])
134
+ height = _positive_number(entry[1])
135
+ else:
136
+ return None
137
+ if width is None or height is None:
138
+ return None
139
+ return int(width), int(height)
140
+
141
+
142
+ def image_count(content: ContentBundle) -> int | None:
143
+ """Number of images available for an image note, when known."""
144
+ asset_count = sum(1 for asset in content.assets if asset.media_type == "image")
145
+ locator_count = 0
146
+ for block in content.blocks:
147
+ locator = block.locator
148
+ if isinstance(locator, ImageRegionLocator):
149
+ locator_count = max(locator_count, locator.image_index)
150
+ known = max(asset_count, locator_count)
151
+ return known or None
152
+
153
+
154
+ def _block_of(content: ContentBundle, block_id: str) -> Any:
155
+ for block in content.blocks:
156
+ if block.block_id == block_id:
157
+ return block
158
+ raise EvidenceValidationError(
159
+ f"references block {block_id!r} that does not exist in revision {content.content_key}"
160
+ )
161
+
162
+
163
+ def _validate_locator_compatibility(
164
+ evidence: EvidenceRefV2,
165
+ ref: ContentRef,
166
+ content: ContentBundle,
167
+ bundle: ExtractionBundleV2,
168
+ ) -> None:
169
+ locator = evidence.locator
170
+ source_id = evidence.source_id
171
+ if is_image_note(ref) and locator.kind in {"time_range", "video_frame"}:
172
+ raise EvidenceValidationError(
173
+ f"evidence {evidence.evidence_id} uses a {locator.kind} locator on image note "
174
+ f"{source_id!r}; image notes have no timeline and must not invent timestamps"
175
+ )
176
+ if isinstance(locator, (TimeRangeLocator, VideoFrameLocator)):
177
+ seconds = locator.time.start if isinstance(locator, TimeRangeLocator) else locator.time
178
+ if seconds < 0:
179
+ raise EvidenceValidationError(
180
+ f"evidence {evidence.evidence_id} cites a negative time ({seconds!r}s)"
181
+ )
182
+ duration = known_duration(content, bundle)
183
+ if duration is not None:
184
+ cited = locator.time.end if isinstance(locator, TimeRangeLocator) else locator.time
185
+ if cited > duration + DURATION_TOLERANCE_SECONDS:
186
+ raise EvidenceValidationError(
187
+ f"evidence {evidence.evidence_id} cites {cited:.3f}s but the known media "
188
+ f"duration is {duration:.3f}s"
189
+ )
190
+ if isinstance(locator, ImageRegionLocator):
191
+ count = image_count(content)
192
+ if count is not None and locator.image_index > count:
193
+ raise EvidenceValidationError(
194
+ f"evidence {evidence.evidence_id} cites image {locator.image_index} but "
195
+ f"{source_id!r} only has {count} image(s)"
196
+ )
197
+ declared = image_dimensions(content, locator.image_index)
198
+ if declared is not None and declared != (locator.image_width, locator.image_height):
199
+ raise EvidenceValidationError(
200
+ f"evidence {evidence.evidence_id} declares image {locator.image_index} as "
201
+ f"{locator.image_width}x{locator.image_height} but the artifact records "
202
+ f"{declared[0]}x{declared[1]}"
203
+ )
204
+ if locator.bbox is not None:
205
+ x0, y0, x1, y1 = locator.bbox
206
+ if not (0 <= x0 < x1 <= locator.image_width and 0 <= y0 < y1 <= locator.image_height):
207
+ raise EvidenceValidationError(
208
+ f"evidence {evidence.evidence_id} has a bbox {locator.bbox!r} outside "
209
+ f"image dimensions {locator.image_width}x{locator.image_height}"
210
+ )
211
+ if isinstance(locator, VideoFrameLocator) and locator.bbox is not None:
212
+ if locator.image_width is None or locator.image_height is None:
213
+ raise EvidenceValidationError(
214
+ f"evidence {evidence.evidence_id} has a frame bbox without frame dimensions"
215
+ )
216
+ if isinstance(locator, ParagraphLocator):
217
+ if locator.paragraph < 0:
218
+ raise EvidenceValidationError(
219
+ f"evidence {evidence.evidence_id} cites a negative paragraph index"
220
+ )
221
+
222
+
223
+ def _validate_quote(evidence: EvidenceRefV2, block: Any) -> None:
224
+ quote = _normalize_text(evidence.quote)
225
+ if not quote:
226
+ raise EvidenceValidationError(f"evidence {evidence.evidence_id} has an empty quote")
227
+ candidates = [block.raw_text]
228
+ if block.cleaned_text:
229
+ candidates.append(block.cleaned_text)
230
+ if not any(quote in _normalize_text(candidate) for candidate in candidates):
231
+ raise EvidenceValidationError(
232
+ f"evidence {evidence.evidence_id} quotes text that does not appear in block "
233
+ f"{block.block_id!r} (fabricated excerpt)"
234
+ )
235
+
236
+
237
+ def validate_evidence(evidence: EvidenceRefV2, bundle: ExtractionBundleV2) -> None:
238
+ """Raise :class:`EvidenceValidationError` when ``evidence`` cannot resolve.
239
+
240
+ Checks, in order: registered source, matching revision content, existing
241
+ block, locator/revision equality (no coordinate drift), locator vs content
242
+ type (image notes reject timestamps), time vs known duration, image index
243
+ vs available images, bbox vs image dimensions, and the quoted excerpt
244
+ actually occurring in the block's text.
245
+ """
246
+ if not isinstance(evidence, EvidenceRefV2):
247
+ raise EvidenceValidationError(
248
+ f"expected an EvidenceRefV2, got {type(evidence).__name__}"
249
+ )
250
+ ref = find_source(bundle, evidence.source_id)
251
+ content = find_content(bundle, evidence.source_id, evidence.source_revision)
252
+ block = _block_of(content, evidence.block_id)
253
+ _validate_locator_compatibility(evidence, ref, content, bundle)
254
+ if locator_key(block.locator) != locator_key(evidence.locator):
255
+ raise EvidenceValidationError(
256
+ f"evidence {evidence.evidence_id} carries locator {locator_key(evidence.locator)} "
257
+ f"but block {block.block_id!r} has {locator_key(block.locator)} (coordinate drift)"
258
+ )
259
+ if block.source_id != evidence.source_id:
260
+ raise EvidenceValidationError(
261
+ f"evidence {evidence.evidence_id} points at a block of another source"
262
+ )
263
+ _validate_quote(evidence, block)
264
+
265
+
266
+ def validate_atom_evidence(
267
+ evidence_ids: Iterable[str], bundle: ExtractionBundleV2, *, atom_id: str = ""
268
+ ) -> list[EvidenceRefV2]:
269
+ """Validate and return the evidence refs referenced by one atom."""
270
+ refs: list[EvidenceRefV2] = []
271
+ for evidence_id in evidence_ids:
272
+ evidence = bundle.evidence_index.get(evidence_id)
273
+ if evidence is None:
274
+ raise EvidenceValidationError(
275
+ f"atom {atom_id or '<unknown>'!r} references unregistered evidence {evidence_id!r}"
276
+ )
277
+ validate_evidence(evidence, bundle)
278
+ refs.append(evidence)
279
+ return refs
280
+
281
+
282
+ def format_evidence(evidence: EvidenceRefV2, ref: ContentRef, *, language: str = "zh") -> str:
283
+ """Format one evidence reference as a portable markdown citation.
284
+
285
+ Time-based content yields the platform's time jump (or the canonical link
286
+ plus readable time text on platforms without one); image notes yield
287
+ ``[图 2](canonical)`` / ``[第 3 段](canonical)``. The quoted excerpt is
288
+ appended verbatim.
289
+ """
290
+ if evidence.source_id != ref.source_id:
291
+ raise EvidenceValidationError(
292
+ f"evidence {evidence.evidence_id} belongs to {evidence.source_id!r}, "
293
+ f"not to {ref.source_id!r}"
294
+ )
295
+ return platform_link(
296
+ ref,
297
+ evidence.locator,
298
+ quote=evidence.quote,
299
+ language=language,
300
+ )
301
+
302
+
303
+ def format_evidence_refs(
304
+ evidence_list: Iterable[EvidenceRefV2],
305
+ bundle: ExtractionBundleV2,
306
+ *,
307
+ language: str = "zh",
308
+ validate: bool = True,
309
+ ) -> list[str]:
310
+ """Validate each evidence ref against ``bundle`` and format it."""
311
+ rendered: list[str] = []
312
+ for evidence in evidence_list:
313
+ if validate:
314
+ validate_evidence(evidence, bundle)
315
+ ref = find_source(bundle, evidence.source_id)
316
+ rendered.append(format_evidence(evidence, ref, language=language))
317
+ return rendered
318
+
319
+
320
+ def evidence_for_atom(
321
+ atom: Any, bundle: ExtractionBundleV2, *, language: str = "zh", validate: bool = True
322
+ ) -> list[str]:
323
+ """Convenience: format every evidence ref of one atom (validated)."""
324
+ evidence_ids = list(getattr(atom, "evidence_ids", []) or [])
325
+ refs = validate_atom_evidence(evidence_ids, bundle, atom_id=getattr(atom, "atom_id", ""))
326
+ return [
327
+ format_evidence(evidence, find_source(bundle, evidence.source_id), language=language)
328
+ for evidence in refs
329
+ ]
330
+
331
+
332
+ def binding_source_ids(evidence_refs: Iterable[EvidenceRefV2]) -> list[str]:
333
+ """Ordered, de-duplicated source ids touched by ``evidence_refs``."""
334
+ seen: list[str] = []
335
+ for evidence in evidence_refs:
336
+ if evidence.source_id not in seen:
337
+ seen.append(evidence.source_id)
338
+ return seen
339
+
340
+
341
+ def locator_summary(evidence: EvidenceRefV2) -> str:
342
+ """Readable locator text without a link (diagnostics / manifests)."""
343
+ return readable_locator(evidence.locator)
344
+
345
+
346
+ __all__ = [
347
+ "DURATION_TOLERANCE_SECONDS",
348
+ "EvidenceValidationError",
349
+ "binding_source_ids",
350
+ "content_ref_for",
351
+ "evidence_for_atom",
352
+ "find_content",
353
+ "find_source",
354
+ "format_evidence",
355
+ "format_evidence_refs",
356
+ "image_count",
357
+ "image_dimensions",
358
+ "known_duration",
359
+ "locator_summary",
360
+ "validate_atom_evidence",
361
+ "validate_evidence",
362
+ ]
@@ -0,0 +1,13 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from pathlib import Path
5
+ from typing import Any
6
+
7
+
8
+ class JsonRenderer:
9
+ def render(self, bundle: Any, output_path: Path, **kwargs: Any) -> None:
10
+ output_path.write_text(
11
+ json.dumps(bundle.model_dump(mode="json"), indent=2, ensure_ascii=False) + "\n",
12
+ encoding="utf-8",
13
+ )
@@ -0,0 +1,18 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from pathlib import Path
5
+ from typing import Any
6
+
7
+ from ytk.models.knowledge import ExtractionBundle
8
+ from ytk.renderers.base import Renderer
9
+
10
+
11
+ class JSONLRenderer(Renderer):
12
+ def render(self, bundle: ExtractionBundle, output_path: Path, **kwargs: Any) -> None:
13
+ output_path.parent.mkdir(parents=True, exist_ok=True)
14
+ lines: list[str] = []
15
+ for atom in bundle.knowledge_atoms:
16
+ obj = atom.model_dump(mode="json")
17
+ lines.append(json.dumps(obj, ensure_ascii=False))
18
+ output_path.write_text("\n".join(lines) + ("\n" if lines else ""), encoding="utf-8")