kd-engine 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kd_engine-1.0.0.dist-info/METADATA +19 -0
- kd_engine-1.0.0.dist-info/RECORD +118 -0
- kd_engine-1.0.0.dist-info/WHEEL +5 -0
- kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
- kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
- kd_engine-1.0.0.dist-info/top_level.txt +1 -0
- ytk/__init__.py +2 -0
- ytk/__main__.py +21 -0
- ytk/batch/__init__.py +39 -0
- ytk/batch/runner.py +1149 -0
- ytk/cache/__init__.py +34 -0
- ytk/cache/v2.py +562 -0
- ytk/cleaning/density.py +40 -0
- ytk/cleaning/fillers.py +18 -0
- ytk/cleaning/sentence_boundary.py +30 -0
- ytk/cleaning/service.py +62 -0
- ytk/cli.py +1281 -0
- ytk/commands/__init__.py +3 -0
- ytk/commands/merge.py +600 -0
- ytk/config.py +757 -0
- ytk/dedup/service.py +166 -0
- ytk/doctor.py +891 -0
- ytk/dotenv.py +334 -0
- ytk/errors.py +38 -0
- ytk/extraction/service.py +451 -0
- ytk/extraction/strategies/base.py +16 -0
- ytk/extraction/strategies/code_block.py +285 -0
- ytk/extraction/strategies/heuristic.py +791 -0
- ytk/extraction/strategies/local_model.py +56 -0
- ytk/extraction/text_context.py +177 -0
- ytk/extraction/visual_context.py +159 -0
- ytk/ingestion/adapters/__init__.py +1 -0
- ytk/ingestion/adapters/_common.py +865 -0
- ytk/ingestion/adapters/base.py +160 -0
- ytk/ingestion/adapters/bilibili.py +271 -0
- ytk/ingestion/adapters/douyin.py +445 -0
- ytk/ingestion/adapters/tiktok.py +102 -0
- ytk/ingestion/adapters/xiaohongshu.py +758 -0
- ytk/ingestion/adapters/youtube.py +74 -0
- ytk/ingestion/browser_acquired.py +220 -0
- ytk/ingestion/http_client.py +43 -0
- ytk/ingestion/local_import.py +202 -0
- ytk/ingestion/media.py +468 -0
- ytk/ingestion/providers/asr.py +902 -0
- ytk/ingestion/providers/base.py +13 -0
- ytk/ingestion/providers/dtk_client.py +816 -0
- ytk/ingestion/providers/minimax.py +350 -0
- ytk/ingestion/providers/transcript_api.py +336 -0
- ytk/ingestion/providers/xhs_client.py +646 -0
- ytk/ingestion/providers/ytdlp.py +400 -0
- ytk/ingestion/registry.py +189 -0
- ytk/ingestion/resolver.py +417 -0
- ytk/ingestion/search.py +56 -0
- ytk/ingestion/service.py +167 -0
- ytk/ingestion/subtitles.py +602 -0
- ytk/ingestion/transcript_cache.py +212 -0
- ytk/ingestion/url_parser.py +53 -0
- ytk/ingestion/video_frames.py +292 -0
- ytk/ingestion/ytdlp_acquire.py +754 -0
- ytk/logging.py +383 -0
- ytk/models/common.py +20 -0
- ytk/models/content.py +451 -0
- ytk/models/knowledge.py +43 -0
- ytk/models/migrations.py +262 -0
- ytk/models/skill.py +13 -0
- ytk/models/source.py +265 -0
- ytk/models/topic.py +18 -0
- ytk/models/transcript.py +30 -0
- ytk/ocr/__init__.py +65 -0
- ytk/ocr/engine.py +245 -0
- ytk/ocr/layout.py +347 -0
- ytk/ocr/pipeline.py +440 -0
- ytk/ocr/rapidocr_engine.py +144 -0
- ytk/pipeline/acquisition_checkpoint.py +240 -0
- ytk/pipeline/artifacts.py +18 -0
- ytk/pipeline/orchestrator.py +598 -0
- ytk/pipeline/processing_identity.py +18 -0
- ytk/pipeline/retry.py +728 -0
- ytk/pipeline/runners.py +578 -0
- ytk/pipeline/state.py +1107 -0
- ytk/pipeline/v2.py +1863 -0
- ytk/py.typed +1 -0
- ytk/quality/__init__.py +9 -0
- ytk/quality/completeness.py +260 -0
- ytk/renderers/base.py +9 -0
- ytk/renderers/citation_alignment.py +617 -0
- ytk/renderers/evidence_formatter.py +362 -0
- ytk/renderers/json_renderer.py +13 -0
- ytk/renderers/jsonl_renderer.py +18 -0
- ytk/renderers/knowledge_doc_renderer.py +1984 -0
- ytk/renderers/markdown_renderer.py +540 -0
- ytk/renderers/obsidian_renderer.py +138 -0
- ytk/renderers/skill_package.py +21 -0
- ytk/renderers/source_links.py +309 -0
- ytk/renderers/summary_renderer.py +163 -0
- ytk/renderers/v2.py +171 -0
- ytk/renderers/v2_jsonl.py +165 -0
- ytk/renderers/v2_markdown.py +477 -0
- ytk/renderers/v2_obsidian.py +113 -0
- ytk/renderers/v2_summary.py +220 -0
- ytk/segmentation/change_points.py +50 -0
- ytk/segmentation/embeddings.py +45 -0
- ytk/segmentation/service.py +318 -0
- ytk/skill_builder/__init__.py +1 -0
- ytk/skill_builder/builder.py +419 -0
- ytk/skill_builder/templates/SKILL.md.j2 +85 -0
- ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
- ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
- ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
- ytk/skill_builder/templates/references/commands.md.j2 +34 -0
- ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
- ytk/skill_builder/templates/references/configs.md.j2 +31 -0
- ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
- ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
- ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
- ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
- ytk/skill_builder/templates.py +29 -0
- ytk/text_normalization.py +44 -0
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""T33.2 — v2 Obsidian renderer.
|
|
2
|
+
|
|
3
|
+
Frontmatter carries the frozen identity fields (``platform`` / ``source_id`` /
|
|
4
|
+
``content_type`` / ``status`` — contract §1.6); the body reuses the v2
|
|
5
|
+
Markdown body so both artifacts cite through the same
|
|
6
|
+
:mod:`ytk.renderers.evidence_formatter` authority. Images are referenced with
|
|
7
|
+
package-relative paths from ``MediaAsset.relative_path``, so the note keeps
|
|
8
|
+
working after the output directory is moved. No access tokens are ever
|
|
9
|
+
written: URLs come from ``ContentRef.canonical_url`` (already stripped).
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import re
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
from ytk.models.content import ExtractionBundleV2
|
|
19
|
+
from ytk.models.source import ContentRef
|
|
20
|
+
from ytk.renderers.markdown_renderer import MarkdownRenderer
|
|
21
|
+
from ytk.renderers.source_links import display_title, platform_label
|
|
22
|
+
from ytk.renderers.v2_markdown import render_body, resolve_language
|
|
23
|
+
|
|
24
|
+
_YAML_SPECIAL_RE = MarkdownRenderer._YAML_SPECIAL_RE
|
|
25
|
+
|
|
26
|
+
_SIMPLE_SAFE_RE = re.compile(r"^[A-Za-z0-9_.:@/-]+$")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _yaml_scalar(value: str, *, quote_always: bool = False) -> str:
|
|
30
|
+
"""Deterministic YAML string: quoted when it could change meaning."""
|
|
31
|
+
text = str(value).replace("\\", "\\\\").replace('"', '\\"').replace("\n", " ")
|
|
32
|
+
if quote_always or not _SIMPLE_SAFE_RE.match(text) or text in {"true", "false", "null", "yes", "no"}:
|
|
33
|
+
return f'"{text}"'
|
|
34
|
+
return text
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def primary_source(bundle: ExtractionBundleV2) -> ContentRef | None:
|
|
38
|
+
return bundle.sources[0] if bundle.sources else None
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def primary_status(bundle: ExtractionBundleV2) -> str:
|
|
42
|
+
if not bundle.contents:
|
|
43
|
+
return "unknown"
|
|
44
|
+
return bundle.contents[0].completeness.status
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def collect_tags(bundle: ExtractionBundleV2) -> list[str]:
|
|
48
|
+
tags: set[str] = set()
|
|
49
|
+
for atom in bundle.knowledge:
|
|
50
|
+
for topic in atom.topic_path:
|
|
51
|
+
tag = MarkdownRenderer.clean_tag(topic)
|
|
52
|
+
if tag:
|
|
53
|
+
tags.add(tag)
|
|
54
|
+
for source in bundle.sources:
|
|
55
|
+
tag = MarkdownRenderer.clean_tag(platform_label(source))
|
|
56
|
+
if tag:
|
|
57
|
+
tags.add(tag)
|
|
58
|
+
return sorted(tags)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def build_frontmatter(bundle: ExtractionBundleV2) -> list[str]:
|
|
62
|
+
source = primary_source(bundle)
|
|
63
|
+
lines: list[str] = ["---"]
|
|
64
|
+
title = bundle.metadata.get("title")
|
|
65
|
+
if not isinstance(title, str) or not title.strip():
|
|
66
|
+
title = display_title(source) if source is not None else "Knowledge"
|
|
67
|
+
lines.append(f"title: {_yaml_scalar(title, quote_always=True)}")
|
|
68
|
+
if source is not None:
|
|
69
|
+
lines.append(f"platform: {_yaml_scalar(source.platform)}")
|
|
70
|
+
lines.append(f"source_id: {_yaml_scalar(source.source_id)}")
|
|
71
|
+
lines.append(f"content_type: {_yaml_scalar(source.content_type)}")
|
|
72
|
+
lines.append(f"url: {_yaml_scalar(source.canonical_url)}")
|
|
73
|
+
lines.append(f"status: {_yaml_scalar(primary_status(bundle))}")
|
|
74
|
+
if len(bundle.sources) > 1:
|
|
75
|
+
lines.append("sources:")
|
|
76
|
+
for ref in bundle.sources:
|
|
77
|
+
lines.append(f" - platform: {_yaml_scalar(ref.platform)}")
|
|
78
|
+
lines.append(f" source_id: {_yaml_scalar(ref.source_id)}")
|
|
79
|
+
lines.append(f" content_type: {_yaml_scalar(ref.content_type)}")
|
|
80
|
+
lines.append(f" url: {_yaml_scalar(ref.canonical_url)}")
|
|
81
|
+
tags = collect_tags(bundle)
|
|
82
|
+
if tags:
|
|
83
|
+
lines.append("tags:")
|
|
84
|
+
for tag in tags:
|
|
85
|
+
lines.append(f" - {tag}")
|
|
86
|
+
lines.append("---")
|
|
87
|
+
return lines
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class V2ObsidianRenderer:
|
|
91
|
+
"""Renderer-protocol compatible v2 Obsidian renderer."""
|
|
92
|
+
|
|
93
|
+
def render(self, bundle: ExtractionBundleV2, output_path: Path, **kwargs: Any) -> None:
|
|
94
|
+
language = resolve_language(kwargs)
|
|
95
|
+
output_path = Path(output_path)
|
|
96
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
97
|
+
lines = build_frontmatter(bundle)
|
|
98
|
+
lines.extend(["", *render_body(bundle, language)])
|
|
99
|
+
output_path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def render_obsidian(bundle: ExtractionBundleV2, output_path: Path, **kwargs: Any) -> None:
|
|
103
|
+
V2ObsidianRenderer().render(bundle, output_path, **kwargs)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
__all__ = [
|
|
107
|
+
"V2ObsidianRenderer",
|
|
108
|
+
"build_frontmatter",
|
|
109
|
+
"collect_tags",
|
|
110
|
+
"primary_source",
|
|
111
|
+
"primary_status",
|
|
112
|
+
"render_obsidian",
|
|
113
|
+
]
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
"""T33.2 — v2 multi-source summary renderer (``summary.md``).
|
|
2
|
+
|
|
3
|
+
Deterministic, offline summary: no LLM, no clock, no environment probing.
|
|
4
|
+
It answers "what is in this merged knowledge product and where did it come
|
|
5
|
+
from" — source count and per-platform provenance, content completeness,
|
|
6
|
+
topic/atom counts and the highest-confidence atoms, each with the same
|
|
7
|
+
citation link used by the other v2 renderers.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from ytk.models.content import ExtractionBundleV2, KnowledgeAtomV2
|
|
16
|
+
from ytk.renderers.evidence_formatter import format_evidence_refs
|
|
17
|
+
from ytk.renderers.source_links import (
|
|
18
|
+
PLATFORM_LABELS,
|
|
19
|
+
display_title,
|
|
20
|
+
platform_label,
|
|
21
|
+
platform_sort_key,
|
|
22
|
+
source_line,
|
|
23
|
+
)
|
|
24
|
+
from ytk.renderers.v2_markdown import (
|
|
25
|
+
CONTENT_TYPE_LABELS,
|
|
26
|
+
atom_type_title,
|
|
27
|
+
bundle_title,
|
|
28
|
+
resolve_language,
|
|
29
|
+
render_processing_notes,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
MAX_SUMMARY_ATOMS = 8
|
|
33
|
+
|
|
34
|
+
_LABELS: dict[str, dict[str, str]] = {
|
|
35
|
+
"zh": {
|
|
36
|
+
"sources": "来源",
|
|
37
|
+
"counts": "规模",
|
|
38
|
+
"platforms": "平台分布",
|
|
39
|
+
"completeness": "内容完整性",
|
|
40
|
+
"highlights": "高置信知识点",
|
|
41
|
+
"atom_one": "知识点",
|
|
42
|
+
"atom_many": "知识点",
|
|
43
|
+
"source_one": "来源",
|
|
44
|
+
"source_many": "来源",
|
|
45
|
+
"content_one": "内容单元",
|
|
46
|
+
"content_many": "内容单元",
|
|
47
|
+
"topic_one": "主题",
|
|
48
|
+
"topic_many": "主题",
|
|
49
|
+
"none": "(无可汇总内容)",
|
|
50
|
+
"note": "*由 knowledge-distiller v2.0 生成的多来源摘要*",
|
|
51
|
+
"evidence": "证据",
|
|
52
|
+
"no_evidence": "(无引用证据)",
|
|
53
|
+
"content_type": "内容类型",
|
|
54
|
+
"confidence": "置信度",
|
|
55
|
+
},
|
|
56
|
+
"en": {
|
|
57
|
+
"sources": "Sources",
|
|
58
|
+
"counts": "Scale",
|
|
59
|
+
"platforms": "Platforms",
|
|
60
|
+
"completeness": "Completeness",
|
|
61
|
+
"highlights": "High-confidence knowledge",
|
|
62
|
+
"atom_one": "atom",
|
|
63
|
+
"atom_many": "atoms",
|
|
64
|
+
"source_one": "source",
|
|
65
|
+
"source_many": "sources",
|
|
66
|
+
"content_one": "content unit",
|
|
67
|
+
"content_many": "content units",
|
|
68
|
+
"topic_one": "topic",
|
|
69
|
+
"topic_many": "topics",
|
|
70
|
+
"none": "(nothing to summarise)",
|
|
71
|
+
"note": "*Multi-source summary generated by knowledge-distiller v2.0*",
|
|
72
|
+
"evidence": "Evidence",
|
|
73
|
+
"no_evidence": "(no evidence captured)",
|
|
74
|
+
"content_type": "Content type",
|
|
75
|
+
"confidence": "Confidence",
|
|
76
|
+
},
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def labels(language: str) -> dict[str, str]:
|
|
81
|
+
return _LABELS["en" if language == "en" else "zh"]
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _count_word(labels_map: dict[str, str], one_key: str, many_key: str, count: int) -> str:
|
|
85
|
+
return labels_map[one_key] if count == 1 else labels_map[many_key]
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def ranked_atoms(bundle: ExtractionBundleV2) -> list[KnowledgeAtomV2]:
|
|
89
|
+
"""Deterministic highlight ranking: confidence desc, then content, then id."""
|
|
90
|
+
return sorted(
|
|
91
|
+
[atom for atom in bundle.knowledge if atom.atom_type != "segment_summary" and atom.confidence >= 0.6],
|
|
92
|
+
key=lambda atom: (-atom.confidence, atom.content.casefold(), atom.atom_id),
|
|
93
|
+
)[:MAX_SUMMARY_ATOMS]
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def platform_counts(bundle: ExtractionBundleV2) -> list[tuple[str, int]]:
|
|
97
|
+
counts: dict[str, int] = {}
|
|
98
|
+
for ref in bundle.sources:
|
|
99
|
+
counts[ref.platform] = counts.get(ref.platform, 0) + 1
|
|
100
|
+
return sorted(counts.items(), key=lambda item: platform_sort_key(item[0]))
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def render_summary_text(bundle: ExtractionBundleV2, language: str) -> str:
|
|
104
|
+
text_labels = labels(language)
|
|
105
|
+
lines: list[str] = [f"# {bundle_title(bundle)}", ""]
|
|
106
|
+
|
|
107
|
+
lines.extend([f"## {text_labels['sources']}", ""])
|
|
108
|
+
if bundle.sources:
|
|
109
|
+
for ref in bundle.sources:
|
|
110
|
+
lines.append(source_line(ref, language=language))
|
|
111
|
+
content_types = sorted(
|
|
112
|
+
{
|
|
113
|
+
CONTENT_TYPE_LABELS.get(ref.content_type, CONTENT_TYPE_LABELS["video"])[
|
|
114
|
+
"en" if language == "en" else "zh"
|
|
115
|
+
]
|
|
116
|
+
for ref in bundle.sources
|
|
117
|
+
}
|
|
118
|
+
)
|
|
119
|
+
lines.append(f"- {text_labels['content_type']}: {', '.join(content_types)}")
|
|
120
|
+
else:
|
|
121
|
+
lines.append(text_labels["none"])
|
|
122
|
+
lines.append("")
|
|
123
|
+
|
|
124
|
+
lines.extend(render_processing_notes(bundle, language))
|
|
125
|
+
|
|
126
|
+
lines.extend([f"## {text_labels['counts']}", ""])
|
|
127
|
+
lines.append(
|
|
128
|
+
f"- {len(bundle.sources)} "
|
|
129
|
+
f"{_count_word(text_labels, 'source_one', 'source_many', len(bundle.sources))}"
|
|
130
|
+
)
|
|
131
|
+
lines.append(
|
|
132
|
+
f"- {len(bundle.contents)} "
|
|
133
|
+
f"{_count_word(text_labels, 'content_one', 'content_many', len(bundle.contents))}"
|
|
134
|
+
)
|
|
135
|
+
lines.append(
|
|
136
|
+
f"- {len(bundle.topics)} "
|
|
137
|
+
f"{_count_word(text_labels, 'topic_one', 'topic_many', len(bundle.topics))}"
|
|
138
|
+
)
|
|
139
|
+
lines.append(
|
|
140
|
+
f"- {len(bundle.knowledge)} "
|
|
141
|
+
f"{_count_word(text_labels, 'atom_one', 'atom_many', len(bundle.knowledge))}"
|
|
142
|
+
)
|
|
143
|
+
lines.append("")
|
|
144
|
+
|
|
145
|
+
lines.extend([f"## {text_labels['platforms']}", ""])
|
|
146
|
+
counts = platform_counts(bundle)
|
|
147
|
+
if counts:
|
|
148
|
+
for platform, count in counts:
|
|
149
|
+
lines.append(f"- {PLATFORM_LABELS.get(platform, platform.title())}: {count}")
|
|
150
|
+
else:
|
|
151
|
+
lines.append(text_labels["none"])
|
|
152
|
+
lines.append("")
|
|
153
|
+
|
|
154
|
+
lines.extend([f"## {text_labels['completeness']}", ""])
|
|
155
|
+
if bundle.contents:
|
|
156
|
+
for content in bundle.contents:
|
|
157
|
+
ref = content.source
|
|
158
|
+
lines.append(
|
|
159
|
+
f"- {platform_label(ref)} · {display_title(ref)}: {content.completeness.status}"
|
|
160
|
+
)
|
|
161
|
+
else:
|
|
162
|
+
lines.append(text_labels["none"])
|
|
163
|
+
lines.append("")
|
|
164
|
+
|
|
165
|
+
lines.extend([f"## {text_labels['highlights']}", ""])
|
|
166
|
+
highlights = ranked_atoms(bundle)
|
|
167
|
+
if not highlights:
|
|
168
|
+
lines.append(text_labels["none"])
|
|
169
|
+
lines.append("")
|
|
170
|
+
for atom in highlights:
|
|
171
|
+
type_title = atom_type_title(atom.atom_type, language)
|
|
172
|
+
lines.append(f"### [{type_title}] {atom.content}")
|
|
173
|
+
lines.append(f"- {text_labels['confidence']}: {atom.confidence:.2f}")
|
|
174
|
+
citations = format_evidence_refs(
|
|
175
|
+
[bundle.evidence_index[eid] for eid in atom.evidence_ids if eid in bundle.evidence_index],
|
|
176
|
+
bundle,
|
|
177
|
+
language=language,
|
|
178
|
+
)
|
|
179
|
+
if citations:
|
|
180
|
+
lines.append(f"- {text_labels['evidence']}: " + "; ".join(citations))
|
|
181
|
+
else:
|
|
182
|
+
lines.append(f"- {text_labels['evidence']}: {text_labels['no_evidence']}")
|
|
183
|
+
lines.append("")
|
|
184
|
+
|
|
185
|
+
excerpts = [atom for atom in bundle.knowledge if atom.atom_type == "segment_summary"]
|
|
186
|
+
if excerpts:
|
|
187
|
+
lines.extend(["## Segment excerpts (not knowledge conclusions)" if language == "en" else "## 分段摘要(非知识结论)", ""])
|
|
188
|
+
for atom in excerpts[:MAX_SUMMARY_ATOMS]:
|
|
189
|
+
lines.append(f"- {atom.content}")
|
|
190
|
+
citations = format_evidence_refs([bundle.evidence_index[eid] for eid in atom.evidence_ids], bundle, language=language)
|
|
191
|
+
lines.append(f" - {text_labels['evidence']}: " + "; ".join(citations))
|
|
192
|
+
lines.append("")
|
|
193
|
+
|
|
194
|
+
lines.extend([text_labels["note"], ""])
|
|
195
|
+
return "\n".join(lines).rstrip() + "\n"
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
class V2SummaryRenderer:
|
|
199
|
+
"""Renderer-protocol compatible v2 summary renderer."""
|
|
200
|
+
|
|
201
|
+
def render(self, bundle: ExtractionBundleV2, output_path: Path, **kwargs: Any) -> None:
|
|
202
|
+
language = resolve_language(kwargs)
|
|
203
|
+
output_path = Path(output_path)
|
|
204
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
205
|
+
output_path.write_text(render_summary_text(bundle, language), encoding="utf-8")
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def render_summary(bundle: ExtractionBundleV2, output_path: Path, **kwargs: Any) -> None:
|
|
209
|
+
V2SummaryRenderer().render(bundle, output_path, **kwargs)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
__all__ = [
|
|
213
|
+
"MAX_SUMMARY_ATOMS",
|
|
214
|
+
"V2SummaryRenderer",
|
|
215
|
+
"labels",
|
|
216
|
+
"platform_counts",
|
|
217
|
+
"ranked_atoms",
|
|
218
|
+
"render_summary",
|
|
219
|
+
"render_summary_text",
|
|
220
|
+
]
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Sequence
|
|
4
|
+
from math import sqrt
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def cosine_similarity(left: Sequence[float], right: Sequence[float]) -> float:
|
|
8
|
+
if len(left) != len(right):
|
|
9
|
+
raise ValueError("Vectors must be the same length.")
|
|
10
|
+
|
|
11
|
+
left_norm = sqrt(sum(value * value for value in left))
|
|
12
|
+
right_norm = sqrt(sum(value * value for value in right))
|
|
13
|
+
if left_norm == 0.0 or right_norm == 0.0:
|
|
14
|
+
return 0.0
|
|
15
|
+
|
|
16
|
+
dot = sum(left_value * right_value for left_value, right_value in zip(left, right, strict=True))
|
|
17
|
+
return dot / (left_norm * right_norm)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def largest_drop_index(
|
|
21
|
+
vectors: Sequence[Sequence[float]],
|
|
22
|
+
*,
|
|
23
|
+
similarity_threshold: float = 0.55,
|
|
24
|
+
) -> int | None:
|
|
25
|
+
candidates = low_similarity_indexes(vectors, similarity_threshold=similarity_threshold)
|
|
26
|
+
return candidates[0] if candidates else None
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def low_similarity_indexes(
|
|
30
|
+
vectors: Sequence[Sequence[float]],
|
|
31
|
+
*,
|
|
32
|
+
similarity_threshold: float = 0.55,
|
|
33
|
+
) -> list[int]:
|
|
34
|
+
if len(vectors) < 2:
|
|
35
|
+
return []
|
|
36
|
+
|
|
37
|
+
similarities = [
|
|
38
|
+
cosine_similarity(vectors[index], vectors[index + 1])
|
|
39
|
+
for index in range(len(vectors) - 1)
|
|
40
|
+
]
|
|
41
|
+
low_points = [
|
|
42
|
+
(index, similarity)
|
|
43
|
+
for index, similarity in enumerate(similarities)
|
|
44
|
+
if similarity < similarity_threshold
|
|
45
|
+
]
|
|
46
|
+
if not low_points:
|
|
47
|
+
return []
|
|
48
|
+
|
|
49
|
+
ranked = sorted(low_points, key=lambda item: item[1])
|
|
50
|
+
return [index for index, _similarity in ranked]
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Sequence
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class SentenceTransformerClient:
|
|
8
|
+
def __init__(
|
|
9
|
+
self,
|
|
10
|
+
model_name: str = "sentence-transformers/all-MiniLM-L6-v2",
|
|
11
|
+
model: Any | None = None,
|
|
12
|
+
) -> None:
|
|
13
|
+
self._model_name = model_name
|
|
14
|
+
self._model = model
|
|
15
|
+
|
|
16
|
+
def encode(self, texts: Sequence[str]) -> list[list[float]]:
|
|
17
|
+
if not texts:
|
|
18
|
+
return []
|
|
19
|
+
|
|
20
|
+
model = self._load_model()
|
|
21
|
+
encoded = model.encode(list(texts))
|
|
22
|
+
return [self._to_float_list(vector) for vector in encoded]
|
|
23
|
+
|
|
24
|
+
def _load_model(self) -> Any:
|
|
25
|
+
if self._model is None:
|
|
26
|
+
try:
|
|
27
|
+
from sentence_transformers import SentenceTransformer
|
|
28
|
+
except ImportError as exc:
|
|
29
|
+
raise RuntimeError(
|
|
30
|
+
"sentence-transformers is required for default topic segmentation embeddings. "
|
|
31
|
+
"Install knowledge-distiller[ml] or inject an embedding client."
|
|
32
|
+
) from exc
|
|
33
|
+
|
|
34
|
+
self._model = SentenceTransformer(self._model_name)
|
|
35
|
+
return self._model
|
|
36
|
+
|
|
37
|
+
@property
|
|
38
|
+
def model_name(self) -> str:
|
|
39
|
+
return self._model_name
|
|
40
|
+
|
|
41
|
+
@staticmethod
|
|
42
|
+
def _to_float_list(vector: Any) -> list[float]:
|
|
43
|
+
if hasattr(vector, "tolist"):
|
|
44
|
+
vector = vector.tolist()
|
|
45
|
+
return [float(value) for value in vector]
|