kd-engine 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kd_engine-1.0.0.dist-info/METADATA +19 -0
- kd_engine-1.0.0.dist-info/RECORD +118 -0
- kd_engine-1.0.0.dist-info/WHEEL +5 -0
- kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
- kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
- kd_engine-1.0.0.dist-info/top_level.txt +1 -0
- ytk/__init__.py +2 -0
- ytk/__main__.py +21 -0
- ytk/batch/__init__.py +39 -0
- ytk/batch/runner.py +1149 -0
- ytk/cache/__init__.py +34 -0
- ytk/cache/v2.py +562 -0
- ytk/cleaning/density.py +40 -0
- ytk/cleaning/fillers.py +18 -0
- ytk/cleaning/sentence_boundary.py +30 -0
- ytk/cleaning/service.py +62 -0
- ytk/cli.py +1281 -0
- ytk/commands/__init__.py +3 -0
- ytk/commands/merge.py +600 -0
- ytk/config.py +757 -0
- ytk/dedup/service.py +166 -0
- ytk/doctor.py +891 -0
- ytk/dotenv.py +334 -0
- ytk/errors.py +38 -0
- ytk/extraction/service.py +451 -0
- ytk/extraction/strategies/base.py +16 -0
- ytk/extraction/strategies/code_block.py +285 -0
- ytk/extraction/strategies/heuristic.py +791 -0
- ytk/extraction/strategies/local_model.py +56 -0
- ytk/extraction/text_context.py +177 -0
- ytk/extraction/visual_context.py +159 -0
- ytk/ingestion/adapters/__init__.py +1 -0
- ytk/ingestion/adapters/_common.py +865 -0
- ytk/ingestion/adapters/base.py +160 -0
- ytk/ingestion/adapters/bilibili.py +271 -0
- ytk/ingestion/adapters/douyin.py +445 -0
- ytk/ingestion/adapters/tiktok.py +102 -0
- ytk/ingestion/adapters/xiaohongshu.py +758 -0
- ytk/ingestion/adapters/youtube.py +74 -0
- ytk/ingestion/browser_acquired.py +220 -0
- ytk/ingestion/http_client.py +43 -0
- ytk/ingestion/local_import.py +202 -0
- ytk/ingestion/media.py +468 -0
- ytk/ingestion/providers/asr.py +902 -0
- ytk/ingestion/providers/base.py +13 -0
- ytk/ingestion/providers/dtk_client.py +816 -0
- ytk/ingestion/providers/minimax.py +350 -0
- ytk/ingestion/providers/transcript_api.py +336 -0
- ytk/ingestion/providers/xhs_client.py +646 -0
- ytk/ingestion/providers/ytdlp.py +400 -0
- ytk/ingestion/registry.py +189 -0
- ytk/ingestion/resolver.py +417 -0
- ytk/ingestion/search.py +56 -0
- ytk/ingestion/service.py +167 -0
- ytk/ingestion/subtitles.py +602 -0
- ytk/ingestion/transcript_cache.py +212 -0
- ytk/ingestion/url_parser.py +53 -0
- ytk/ingestion/video_frames.py +292 -0
- ytk/ingestion/ytdlp_acquire.py +754 -0
- ytk/logging.py +383 -0
- ytk/models/common.py +20 -0
- ytk/models/content.py +451 -0
- ytk/models/knowledge.py +43 -0
- ytk/models/migrations.py +262 -0
- ytk/models/skill.py +13 -0
- ytk/models/source.py +265 -0
- ytk/models/topic.py +18 -0
- ytk/models/transcript.py +30 -0
- ytk/ocr/__init__.py +65 -0
- ytk/ocr/engine.py +245 -0
- ytk/ocr/layout.py +347 -0
- ytk/ocr/pipeline.py +440 -0
- ytk/ocr/rapidocr_engine.py +144 -0
- ytk/pipeline/acquisition_checkpoint.py +240 -0
- ytk/pipeline/artifacts.py +18 -0
- ytk/pipeline/orchestrator.py +598 -0
- ytk/pipeline/processing_identity.py +18 -0
- ytk/pipeline/retry.py +728 -0
- ytk/pipeline/runners.py +578 -0
- ytk/pipeline/state.py +1107 -0
- ytk/pipeline/v2.py +1863 -0
- ytk/py.typed +1 -0
- ytk/quality/__init__.py +9 -0
- ytk/quality/completeness.py +260 -0
- ytk/renderers/base.py +9 -0
- ytk/renderers/citation_alignment.py +617 -0
- ytk/renderers/evidence_formatter.py +362 -0
- ytk/renderers/json_renderer.py +13 -0
- ytk/renderers/jsonl_renderer.py +18 -0
- ytk/renderers/knowledge_doc_renderer.py +1984 -0
- ytk/renderers/markdown_renderer.py +540 -0
- ytk/renderers/obsidian_renderer.py +138 -0
- ytk/renderers/skill_package.py +21 -0
- ytk/renderers/source_links.py +309 -0
- ytk/renderers/summary_renderer.py +163 -0
- ytk/renderers/v2.py +171 -0
- ytk/renderers/v2_jsonl.py +165 -0
- ytk/renderers/v2_markdown.py +477 -0
- ytk/renderers/v2_obsidian.py +113 -0
- ytk/renderers/v2_summary.py +220 -0
- ytk/segmentation/change_points.py +50 -0
- ytk/segmentation/embeddings.py +45 -0
- ytk/segmentation/service.py +318 -0
- ytk/skill_builder/__init__.py +1 -0
- ytk/skill_builder/builder.py +419 -0
- ytk/skill_builder/templates/SKILL.md.j2 +85 -0
- ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
- ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
- ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
- ytk/skill_builder/templates/references/commands.md.j2 +34 -0
- ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
- ytk/skill_builder/templates/references/configs.md.j2 +31 -0
- ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
- ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
- ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
- ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
- ytk/skill_builder/templates.py +29 -0
- ytk/text_normalization.py +44 -0
|
@@ -0,0 +1,362 @@
|
|
|
1
|
+
"""T33.1 — the single evidence formatting / validation authority for v2.
|
|
2
|
+
|
|
3
|
+
Everything a renderer writes about evidence (link, readable locator, quoted
|
|
4
|
+
excerpt) is produced here; nothing downstream formats ``evidence.start`` /
|
|
5
|
+
``evidence.end`` by hand any more. The same module is the *validator*: a
|
|
6
|
+
citation that does not resolve to a real block of the matching source
|
|
7
|
+
revision, claims an image that does not exist, carries a bounding box outside
|
|
8
|
+
the image, cites time beyond the media duration, or invents a timestamp for an
|
|
9
|
+
image note is rejected with a clear error instead of being rendered (A22).
|
|
10
|
+
|
|
11
|
+
Design rules (plan §4.3, contract §1.6):
|
|
12
|
+
|
|
13
|
+
* image notes never get ``[00:00]``-style citations;
|
|
14
|
+
* ``locator`` on an evidence ref must equal the locator of the block it points
|
|
15
|
+
at — a mismatching coordinate is a fabrication, not "close enough";
|
|
16
|
+
* the quoted excerpt must be present in the block's raw (or cleaned) text;
|
|
17
|
+
whitespace differences are tolerated, re-wording is not;
|
|
18
|
+
* validation is deterministic and offline (no network, no model).
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import re
|
|
24
|
+
from collections.abc import Iterable, Sequence
|
|
25
|
+
from typing import Any
|
|
26
|
+
|
|
27
|
+
from ytk.errors import UserFacingError
|
|
28
|
+
from ytk.models.content import (
|
|
29
|
+
ContentBundle,
|
|
30
|
+
EvidenceRefV2,
|
|
31
|
+
ExtractionBundleV2,
|
|
32
|
+
ImageRegionLocator,
|
|
33
|
+
ParagraphLocator,
|
|
34
|
+
TimeRangeLocator,
|
|
35
|
+
VideoFrameLocator,
|
|
36
|
+
locator_key,
|
|
37
|
+
)
|
|
38
|
+
from ytk.models.source import ContentRef
|
|
39
|
+
from ytk.renderers.source_links import (
|
|
40
|
+
is_image_note,
|
|
41
|
+
platform_link,
|
|
42
|
+
readable_locator,
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
Locator = TimeRangeLocator | ParagraphLocator | ImageRegionLocator | VideoFrameLocator
|
|
46
|
+
|
|
47
|
+
#: Time citations past the media duration are rejected beyond this slack.
|
|
48
|
+
DURATION_TOLERANCE_SECONDS = 1.0
|
|
49
|
+
|
|
50
|
+
_WHITESPACE_RE = re.compile(r"\s+")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class EvidenceValidationError(UserFacingError):
|
|
54
|
+
"""An evidence reference is fabricated or cannot resolve to its source."""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _normalize_text(text: str) -> str:
|
|
58
|
+
return _WHITESPACE_RE.sub(" ", str(text)).strip()
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def find_source(bundle: ExtractionBundleV2, source_id: str) -> ContentRef:
|
|
62
|
+
"""Return the registered :class:`ContentRef` for ``source_id``."""
|
|
63
|
+
for ref in bundle.sources:
|
|
64
|
+
if ref.source_id == source_id:
|
|
65
|
+
return ref
|
|
66
|
+
raise EvidenceValidationError(
|
|
67
|
+
f"evidence references source {source_id!r} that is not registered in the bundle"
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def find_content(bundle: ExtractionBundleV2, source_id: str, source_revision: str) -> ContentBundle:
|
|
72
|
+
"""Return the content bundle for ``(source_id, source_revision)``."""
|
|
73
|
+
for content in bundle.contents:
|
|
74
|
+
if content.source.source_id == source_id and content.source_revision == source_revision:
|
|
75
|
+
return content
|
|
76
|
+
raise EvidenceValidationError(
|
|
77
|
+
f"evidence references revision {source_id}@{source_revision} that is not present in contents"
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def content_ref_for(content: ContentBundle) -> ContentRef:
|
|
82
|
+
return content.source
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _positive_number(value: Any) -> float | None:
|
|
86
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
87
|
+
return None
|
|
88
|
+
if value <= 0:
|
|
89
|
+
return None
|
|
90
|
+
return float(value)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def known_duration(content: ContentBundle, bundle: ExtractionBundleV2 | None = None) -> float | None:
|
|
94
|
+
"""Media duration in seconds when the artifacts record it.
|
|
95
|
+
|
|
96
|
+
Checked (in order): ``content.metadata["duration"]``,
|
|
97
|
+
``content.metadata["duration_seconds"]``, the same keys on the bundle
|
|
98
|
+
metadata, then the largest timestamp carried by a time locator. Unknown
|
|
99
|
+
duration means "cannot check" — never a licence to accept any value.
|
|
100
|
+
"""
|
|
101
|
+
for payload in (content.metadata, (bundle.metadata if bundle is not None else {})):
|
|
102
|
+
for key in ("duration", "duration_seconds", "media_duration"):
|
|
103
|
+
value = _positive_number(payload.get(key)) if isinstance(payload, dict) else None
|
|
104
|
+
if value is not None:
|
|
105
|
+
return value
|
|
106
|
+
latest: float | None = None
|
|
107
|
+
for block in content.blocks:
|
|
108
|
+
locator = block.locator
|
|
109
|
+
if isinstance(locator, TimeRangeLocator):
|
|
110
|
+
latest = locator.time.end if latest is None else max(latest, locator.time.end)
|
|
111
|
+
elif isinstance(locator, VideoFrameLocator):
|
|
112
|
+
latest = locator.time if latest is None else max(latest, locator.time)
|
|
113
|
+
return latest
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def image_dimensions(content: ContentBundle, image_index: int) -> tuple[int, int] | None:
|
|
117
|
+
"""Declared ``(width, height)`` for ``image_index`` when the artifacts say.
|
|
118
|
+
|
|
119
|
+
Accepts ``metadata["image_sizes"]`` as a list of ``[w, h]`` pairs or of
|
|
120
|
+
``{"width": w, "height": h}`` dicts (1-based index order). Returns
|
|
121
|
+
``None`` when the artifact does not record dimensions.
|
|
122
|
+
"""
|
|
123
|
+
sizes = content.metadata.get("image_sizes") if isinstance(content.metadata, dict) else None
|
|
124
|
+
if not isinstance(sizes, Sequence) or not sizes:
|
|
125
|
+
return None
|
|
126
|
+
if image_index < 1 or image_index > len(sizes):
|
|
127
|
+
return None
|
|
128
|
+
entry = sizes[image_index - 1]
|
|
129
|
+
if isinstance(entry, dict):
|
|
130
|
+
width = _positive_number(entry.get("width"))
|
|
131
|
+
height = _positive_number(entry.get("height"))
|
|
132
|
+
elif isinstance(entry, Sequence) and len(entry) == 2:
|
|
133
|
+
width = _positive_number(entry[0])
|
|
134
|
+
height = _positive_number(entry[1])
|
|
135
|
+
else:
|
|
136
|
+
return None
|
|
137
|
+
if width is None or height is None:
|
|
138
|
+
return None
|
|
139
|
+
return int(width), int(height)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def image_count(content: ContentBundle) -> int | None:
|
|
143
|
+
"""Number of images available for an image note, when known."""
|
|
144
|
+
asset_count = sum(1 for asset in content.assets if asset.media_type == "image")
|
|
145
|
+
locator_count = 0
|
|
146
|
+
for block in content.blocks:
|
|
147
|
+
locator = block.locator
|
|
148
|
+
if isinstance(locator, ImageRegionLocator):
|
|
149
|
+
locator_count = max(locator_count, locator.image_index)
|
|
150
|
+
known = max(asset_count, locator_count)
|
|
151
|
+
return known or None
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _block_of(content: ContentBundle, block_id: str) -> Any:
|
|
155
|
+
for block in content.blocks:
|
|
156
|
+
if block.block_id == block_id:
|
|
157
|
+
return block
|
|
158
|
+
raise EvidenceValidationError(
|
|
159
|
+
f"references block {block_id!r} that does not exist in revision {content.content_key}"
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _validate_locator_compatibility(
|
|
164
|
+
evidence: EvidenceRefV2,
|
|
165
|
+
ref: ContentRef,
|
|
166
|
+
content: ContentBundle,
|
|
167
|
+
bundle: ExtractionBundleV2,
|
|
168
|
+
) -> None:
|
|
169
|
+
locator = evidence.locator
|
|
170
|
+
source_id = evidence.source_id
|
|
171
|
+
if is_image_note(ref) and locator.kind in {"time_range", "video_frame"}:
|
|
172
|
+
raise EvidenceValidationError(
|
|
173
|
+
f"evidence {evidence.evidence_id} uses a {locator.kind} locator on image note "
|
|
174
|
+
f"{source_id!r}; image notes have no timeline and must not invent timestamps"
|
|
175
|
+
)
|
|
176
|
+
if isinstance(locator, (TimeRangeLocator, VideoFrameLocator)):
|
|
177
|
+
seconds = locator.time.start if isinstance(locator, TimeRangeLocator) else locator.time
|
|
178
|
+
if seconds < 0:
|
|
179
|
+
raise EvidenceValidationError(
|
|
180
|
+
f"evidence {evidence.evidence_id} cites a negative time ({seconds!r}s)"
|
|
181
|
+
)
|
|
182
|
+
duration = known_duration(content, bundle)
|
|
183
|
+
if duration is not None:
|
|
184
|
+
cited = locator.time.end if isinstance(locator, TimeRangeLocator) else locator.time
|
|
185
|
+
if cited > duration + DURATION_TOLERANCE_SECONDS:
|
|
186
|
+
raise EvidenceValidationError(
|
|
187
|
+
f"evidence {evidence.evidence_id} cites {cited:.3f}s but the known media "
|
|
188
|
+
f"duration is {duration:.3f}s"
|
|
189
|
+
)
|
|
190
|
+
if isinstance(locator, ImageRegionLocator):
|
|
191
|
+
count = image_count(content)
|
|
192
|
+
if count is not None and locator.image_index > count:
|
|
193
|
+
raise EvidenceValidationError(
|
|
194
|
+
f"evidence {evidence.evidence_id} cites image {locator.image_index} but "
|
|
195
|
+
f"{source_id!r} only has {count} image(s)"
|
|
196
|
+
)
|
|
197
|
+
declared = image_dimensions(content, locator.image_index)
|
|
198
|
+
if declared is not None and declared != (locator.image_width, locator.image_height):
|
|
199
|
+
raise EvidenceValidationError(
|
|
200
|
+
f"evidence {evidence.evidence_id} declares image {locator.image_index} as "
|
|
201
|
+
f"{locator.image_width}x{locator.image_height} but the artifact records "
|
|
202
|
+
f"{declared[0]}x{declared[1]}"
|
|
203
|
+
)
|
|
204
|
+
if locator.bbox is not None:
|
|
205
|
+
x0, y0, x1, y1 = locator.bbox
|
|
206
|
+
if not (0 <= x0 < x1 <= locator.image_width and 0 <= y0 < y1 <= locator.image_height):
|
|
207
|
+
raise EvidenceValidationError(
|
|
208
|
+
f"evidence {evidence.evidence_id} has a bbox {locator.bbox!r} outside "
|
|
209
|
+
f"image dimensions {locator.image_width}x{locator.image_height}"
|
|
210
|
+
)
|
|
211
|
+
if isinstance(locator, VideoFrameLocator) and locator.bbox is not None:
|
|
212
|
+
if locator.image_width is None or locator.image_height is None:
|
|
213
|
+
raise EvidenceValidationError(
|
|
214
|
+
f"evidence {evidence.evidence_id} has a frame bbox without frame dimensions"
|
|
215
|
+
)
|
|
216
|
+
if isinstance(locator, ParagraphLocator):
|
|
217
|
+
if locator.paragraph < 0:
|
|
218
|
+
raise EvidenceValidationError(
|
|
219
|
+
f"evidence {evidence.evidence_id} cites a negative paragraph index"
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _validate_quote(evidence: EvidenceRefV2, block: Any) -> None:
|
|
224
|
+
quote = _normalize_text(evidence.quote)
|
|
225
|
+
if not quote:
|
|
226
|
+
raise EvidenceValidationError(f"evidence {evidence.evidence_id} has an empty quote")
|
|
227
|
+
candidates = [block.raw_text]
|
|
228
|
+
if block.cleaned_text:
|
|
229
|
+
candidates.append(block.cleaned_text)
|
|
230
|
+
if not any(quote in _normalize_text(candidate) for candidate in candidates):
|
|
231
|
+
raise EvidenceValidationError(
|
|
232
|
+
f"evidence {evidence.evidence_id} quotes text that does not appear in block "
|
|
233
|
+
f"{block.block_id!r} (fabricated excerpt)"
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def validate_evidence(evidence: EvidenceRefV2, bundle: ExtractionBundleV2) -> None:
|
|
238
|
+
"""Raise :class:`EvidenceValidationError` when ``evidence`` cannot resolve.
|
|
239
|
+
|
|
240
|
+
Checks, in order: registered source, matching revision content, existing
|
|
241
|
+
block, locator/revision equality (no coordinate drift), locator vs content
|
|
242
|
+
type (image notes reject timestamps), time vs known duration, image index
|
|
243
|
+
vs available images, bbox vs image dimensions, and the quoted excerpt
|
|
244
|
+
actually occurring in the block's text.
|
|
245
|
+
"""
|
|
246
|
+
if not isinstance(evidence, EvidenceRefV2):
|
|
247
|
+
raise EvidenceValidationError(
|
|
248
|
+
f"expected an EvidenceRefV2, got {type(evidence).__name__}"
|
|
249
|
+
)
|
|
250
|
+
ref = find_source(bundle, evidence.source_id)
|
|
251
|
+
content = find_content(bundle, evidence.source_id, evidence.source_revision)
|
|
252
|
+
block = _block_of(content, evidence.block_id)
|
|
253
|
+
_validate_locator_compatibility(evidence, ref, content, bundle)
|
|
254
|
+
if locator_key(block.locator) != locator_key(evidence.locator):
|
|
255
|
+
raise EvidenceValidationError(
|
|
256
|
+
f"evidence {evidence.evidence_id} carries locator {locator_key(evidence.locator)} "
|
|
257
|
+
f"but block {block.block_id!r} has {locator_key(block.locator)} (coordinate drift)"
|
|
258
|
+
)
|
|
259
|
+
if block.source_id != evidence.source_id:
|
|
260
|
+
raise EvidenceValidationError(
|
|
261
|
+
f"evidence {evidence.evidence_id} points at a block of another source"
|
|
262
|
+
)
|
|
263
|
+
_validate_quote(evidence, block)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def validate_atom_evidence(
|
|
267
|
+
evidence_ids: Iterable[str], bundle: ExtractionBundleV2, *, atom_id: str = ""
|
|
268
|
+
) -> list[EvidenceRefV2]:
|
|
269
|
+
"""Validate and return the evidence refs referenced by one atom."""
|
|
270
|
+
refs: list[EvidenceRefV2] = []
|
|
271
|
+
for evidence_id in evidence_ids:
|
|
272
|
+
evidence = bundle.evidence_index.get(evidence_id)
|
|
273
|
+
if evidence is None:
|
|
274
|
+
raise EvidenceValidationError(
|
|
275
|
+
f"atom {atom_id or '<unknown>'!r} references unregistered evidence {evidence_id!r}"
|
|
276
|
+
)
|
|
277
|
+
validate_evidence(evidence, bundle)
|
|
278
|
+
refs.append(evidence)
|
|
279
|
+
return refs
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def format_evidence(evidence: EvidenceRefV2, ref: ContentRef, *, language: str = "zh") -> str:
|
|
283
|
+
"""Format one evidence reference as a portable markdown citation.
|
|
284
|
+
|
|
285
|
+
Time-based content yields the platform's time jump (or the canonical link
|
|
286
|
+
plus readable time text on platforms without one); image notes yield
|
|
287
|
+
``[图 2](canonical)`` / ``[第 3 段](canonical)``. The quoted excerpt is
|
|
288
|
+
appended verbatim.
|
|
289
|
+
"""
|
|
290
|
+
if evidence.source_id != ref.source_id:
|
|
291
|
+
raise EvidenceValidationError(
|
|
292
|
+
f"evidence {evidence.evidence_id} belongs to {evidence.source_id!r}, "
|
|
293
|
+
f"not to {ref.source_id!r}"
|
|
294
|
+
)
|
|
295
|
+
return platform_link(
|
|
296
|
+
ref,
|
|
297
|
+
evidence.locator,
|
|
298
|
+
quote=evidence.quote,
|
|
299
|
+
language=language,
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def format_evidence_refs(
|
|
304
|
+
evidence_list: Iterable[EvidenceRefV2],
|
|
305
|
+
bundle: ExtractionBundleV2,
|
|
306
|
+
*,
|
|
307
|
+
language: str = "zh",
|
|
308
|
+
validate: bool = True,
|
|
309
|
+
) -> list[str]:
|
|
310
|
+
"""Validate each evidence ref against ``bundle`` and format it."""
|
|
311
|
+
rendered: list[str] = []
|
|
312
|
+
for evidence in evidence_list:
|
|
313
|
+
if validate:
|
|
314
|
+
validate_evidence(evidence, bundle)
|
|
315
|
+
ref = find_source(bundle, evidence.source_id)
|
|
316
|
+
rendered.append(format_evidence(evidence, ref, language=language))
|
|
317
|
+
return rendered
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def evidence_for_atom(
|
|
321
|
+
atom: Any, bundle: ExtractionBundleV2, *, language: str = "zh", validate: bool = True
|
|
322
|
+
) -> list[str]:
|
|
323
|
+
"""Convenience: format every evidence ref of one atom (validated)."""
|
|
324
|
+
evidence_ids = list(getattr(atom, "evidence_ids", []) or [])
|
|
325
|
+
refs = validate_atom_evidence(evidence_ids, bundle, atom_id=getattr(atom, "atom_id", ""))
|
|
326
|
+
return [
|
|
327
|
+
format_evidence(evidence, find_source(bundle, evidence.source_id), language=language)
|
|
328
|
+
for evidence in refs
|
|
329
|
+
]
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def binding_source_ids(evidence_refs: Iterable[EvidenceRefV2]) -> list[str]:
|
|
333
|
+
"""Ordered, de-duplicated source ids touched by ``evidence_refs``."""
|
|
334
|
+
seen: list[str] = []
|
|
335
|
+
for evidence in evidence_refs:
|
|
336
|
+
if evidence.source_id not in seen:
|
|
337
|
+
seen.append(evidence.source_id)
|
|
338
|
+
return seen
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def locator_summary(evidence: EvidenceRefV2) -> str:
|
|
342
|
+
"""Readable locator text without a link (diagnostics / manifests)."""
|
|
343
|
+
return readable_locator(evidence.locator)
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
__all__ = [
|
|
347
|
+
"DURATION_TOLERANCE_SECONDS",
|
|
348
|
+
"EvidenceValidationError",
|
|
349
|
+
"binding_source_ids",
|
|
350
|
+
"content_ref_for",
|
|
351
|
+
"evidence_for_atom",
|
|
352
|
+
"find_content",
|
|
353
|
+
"find_source",
|
|
354
|
+
"format_evidence",
|
|
355
|
+
"format_evidence_refs",
|
|
356
|
+
"image_count",
|
|
357
|
+
"image_dimensions",
|
|
358
|
+
"known_duration",
|
|
359
|
+
"locator_summary",
|
|
360
|
+
"validate_atom_evidence",
|
|
361
|
+
"validate_evidence",
|
|
362
|
+
]
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class JsonRenderer:
|
|
9
|
+
def render(self, bundle: Any, output_path: Path, **kwargs: Any) -> None:
|
|
10
|
+
output_path.write_text(
|
|
11
|
+
json.dumps(bundle.model_dump(mode="json"), indent=2, ensure_ascii=False) + "\n",
|
|
12
|
+
encoding="utf-8",
|
|
13
|
+
)
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from ytk.models.knowledge import ExtractionBundle
|
|
8
|
+
from ytk.renderers.base import Renderer
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class JSONLRenderer(Renderer):
|
|
12
|
+
def render(self, bundle: ExtractionBundle, output_path: Path, **kwargs: Any) -> None:
|
|
13
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
14
|
+
lines: list[str] = []
|
|
15
|
+
for atom in bundle.knowledge_atoms:
|
|
16
|
+
obj = atom.model_dump(mode="json")
|
|
17
|
+
lines.append(json.dumps(obj, ensure_ascii=False))
|
|
18
|
+
output_path.write_text("\n".join(lines) + ("\n" if lines else ""), encoding="utf-8")
|