kd-engine 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kd_engine-1.0.0.dist-info/METADATA +19 -0
- kd_engine-1.0.0.dist-info/RECORD +118 -0
- kd_engine-1.0.0.dist-info/WHEEL +5 -0
- kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
- kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
- kd_engine-1.0.0.dist-info/top_level.txt +1 -0
- ytk/__init__.py +2 -0
- ytk/__main__.py +21 -0
- ytk/batch/__init__.py +39 -0
- ytk/batch/runner.py +1149 -0
- ytk/cache/__init__.py +34 -0
- ytk/cache/v2.py +562 -0
- ytk/cleaning/density.py +40 -0
- ytk/cleaning/fillers.py +18 -0
- ytk/cleaning/sentence_boundary.py +30 -0
- ytk/cleaning/service.py +62 -0
- ytk/cli.py +1281 -0
- ytk/commands/__init__.py +3 -0
- ytk/commands/merge.py +600 -0
- ytk/config.py +757 -0
- ytk/dedup/service.py +166 -0
- ytk/doctor.py +891 -0
- ytk/dotenv.py +334 -0
- ytk/errors.py +38 -0
- ytk/extraction/service.py +451 -0
- ytk/extraction/strategies/base.py +16 -0
- ytk/extraction/strategies/code_block.py +285 -0
- ytk/extraction/strategies/heuristic.py +791 -0
- ytk/extraction/strategies/local_model.py +56 -0
- ytk/extraction/text_context.py +177 -0
- ytk/extraction/visual_context.py +159 -0
- ytk/ingestion/adapters/__init__.py +1 -0
- ytk/ingestion/adapters/_common.py +865 -0
- ytk/ingestion/adapters/base.py +160 -0
- ytk/ingestion/adapters/bilibili.py +271 -0
- ytk/ingestion/adapters/douyin.py +445 -0
- ytk/ingestion/adapters/tiktok.py +102 -0
- ytk/ingestion/adapters/xiaohongshu.py +758 -0
- ytk/ingestion/adapters/youtube.py +74 -0
- ytk/ingestion/browser_acquired.py +220 -0
- ytk/ingestion/http_client.py +43 -0
- ytk/ingestion/local_import.py +202 -0
- ytk/ingestion/media.py +468 -0
- ytk/ingestion/providers/asr.py +902 -0
- ytk/ingestion/providers/base.py +13 -0
- ytk/ingestion/providers/dtk_client.py +816 -0
- ytk/ingestion/providers/minimax.py +350 -0
- ytk/ingestion/providers/transcript_api.py +336 -0
- ytk/ingestion/providers/xhs_client.py +646 -0
- ytk/ingestion/providers/ytdlp.py +400 -0
- ytk/ingestion/registry.py +189 -0
- ytk/ingestion/resolver.py +417 -0
- ytk/ingestion/search.py +56 -0
- ytk/ingestion/service.py +167 -0
- ytk/ingestion/subtitles.py +602 -0
- ytk/ingestion/transcript_cache.py +212 -0
- ytk/ingestion/url_parser.py +53 -0
- ytk/ingestion/video_frames.py +292 -0
- ytk/ingestion/ytdlp_acquire.py +754 -0
- ytk/logging.py +383 -0
- ytk/models/common.py +20 -0
- ytk/models/content.py +451 -0
- ytk/models/knowledge.py +43 -0
- ytk/models/migrations.py +262 -0
- ytk/models/skill.py +13 -0
- ytk/models/source.py +265 -0
- ytk/models/topic.py +18 -0
- ytk/models/transcript.py +30 -0
- ytk/ocr/__init__.py +65 -0
- ytk/ocr/engine.py +245 -0
- ytk/ocr/layout.py +347 -0
- ytk/ocr/pipeline.py +440 -0
- ytk/ocr/rapidocr_engine.py +144 -0
- ytk/pipeline/acquisition_checkpoint.py +240 -0
- ytk/pipeline/artifacts.py +18 -0
- ytk/pipeline/orchestrator.py +598 -0
- ytk/pipeline/processing_identity.py +18 -0
- ytk/pipeline/retry.py +728 -0
- ytk/pipeline/runners.py +578 -0
- ytk/pipeline/state.py +1107 -0
- ytk/pipeline/v2.py +1863 -0
- ytk/py.typed +1 -0
- ytk/quality/__init__.py +9 -0
- ytk/quality/completeness.py +260 -0
- ytk/renderers/base.py +9 -0
- ytk/renderers/citation_alignment.py +617 -0
- ytk/renderers/evidence_formatter.py +362 -0
- ytk/renderers/json_renderer.py +13 -0
- ytk/renderers/jsonl_renderer.py +18 -0
- ytk/renderers/knowledge_doc_renderer.py +1984 -0
- ytk/renderers/markdown_renderer.py +540 -0
- ytk/renderers/obsidian_renderer.py +138 -0
- ytk/renderers/skill_package.py +21 -0
- ytk/renderers/source_links.py +309 -0
- ytk/renderers/summary_renderer.py +163 -0
- ytk/renderers/v2.py +171 -0
- ytk/renderers/v2_jsonl.py +165 -0
- ytk/renderers/v2_markdown.py +477 -0
- ytk/renderers/v2_obsidian.py +113 -0
- ytk/renderers/v2_summary.py +220 -0
- ytk/segmentation/change_points.py +50 -0
- ytk/segmentation/embeddings.py +45 -0
- ytk/segmentation/service.py +318 -0
- ytk/skill_builder/__init__.py +1 -0
- ytk/skill_builder/builder.py +419 -0
- ytk/skill_builder/templates/SKILL.md.j2 +85 -0
- ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
- ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
- ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
- ytk/skill_builder/templates/references/commands.md.j2 +34 -0
- ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
- ytk/skill_builder/templates/references/configs.md.j2 +31 -0
- ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
- ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
- ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
- ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
- ytk/skill_builder/templates.py +29 -0
- ytk/text_normalization.py +44 -0
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def main() -> None:
|
|
5
|
+
target = Path(".")
|
|
6
|
+
print(
|
|
7
|
+
"This package was generated from extraction artifacts. "
|
|
8
|
+
"Rebuild it from source data by calling SkillBuilder with the original bundle."
|
|
9
|
+
)
|
|
10
|
+
print(f"Package root: {target.resolve()}")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
if __name__ == "__main__":
|
|
14
|
+
main()
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Validate the structure of a generated skill package.
|
|
3
|
+
|
|
4
|
+
Run from inside the skill directory (the parent of ``scripts/``):
|
|
5
|
+
|
|
6
|
+
python3 scripts/validate_output.py
|
|
7
|
+
|
|
8
|
+
The checks mirror the ``evals/evals.json`` contract emitted alongside this
|
|
9
|
+
script, so a generated package can self-verify before another agent consumes
|
|
10
|
+
it. Keep the two in sync if you add new structural rules.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import re
|
|
17
|
+
import sys
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
SKILL_DIR = Path(__file__).resolve().parent.parent
|
|
22
|
+
SKILL_MD = SKILL_DIR / "SKILL.md"
|
|
23
|
+
EVALS_JSON = SKILL_DIR / "evals" / "evals.json"
|
|
24
|
+
|
|
25
|
+
REQUIRED_FILES = (
|
|
26
|
+
"SKILL.md",
|
|
27
|
+
"references/source-summary.md",
|
|
28
|
+
"references/commands.md",
|
|
29
|
+
"references/configs.md",
|
|
30
|
+
"references/concepts.md",
|
|
31
|
+
"references/evidence-policy.md",
|
|
32
|
+
"examples/example-input.md",
|
|
33
|
+
"examples/example-output.md",
|
|
34
|
+
"evals/evals.json",
|
|
35
|
+
"scripts/build_skill.py",
|
|
36
|
+
"scripts/validate_output.py",
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
REFERENCE_FILES = (
|
|
40
|
+
"references/source-summary.md",
|
|
41
|
+
"references/commands.md",
|
|
42
|
+
"references/configs.md",
|
|
43
|
+
"references/concepts.md",
|
|
44
|
+
"references/evidence-policy.md",
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
# Matches a YAML frontmatter name field. The skill-creator linter enforces
|
|
48
|
+
# that this matches the directory name, so we mirror that rule locally.
|
|
49
|
+
_FRONTMATTER_NAME_RE = re.compile(r"^---\s*\nname:\s*(\S+)\s*$", re.MULTILINE)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _read(path: Path) -> str:
|
|
53
|
+
if not path.exists():
|
|
54
|
+
return ""
|
|
55
|
+
return path.read_text(encoding="utf-8")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def check_required_files() -> list[str]:
|
|
59
|
+
"""All hand-authored and rendered files must be present."""
|
|
60
|
+
missing = [name for name in REQUIRED_FILES if not (SKILL_DIR / name).exists()]
|
|
61
|
+
return [f"missing required file: {name}" for name in missing]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def check_skill_name_matches_dir() -> list[str]:
|
|
65
|
+
"""SKILL.md frontmatter name must equal the directory name."""
|
|
66
|
+
if not SKILL_MD.exists():
|
|
67
|
+
return []
|
|
68
|
+
match = _FRONTMATTER_NAME_RE.search(_read(SKILL_MD))
|
|
69
|
+
if not match:
|
|
70
|
+
return ["SKILL.md frontmatter is missing the `name` field"]
|
|
71
|
+
declared = match.group(1)
|
|
72
|
+
expected = SKILL_DIR.name
|
|
73
|
+
if declared != expected:
|
|
74
|
+
return [
|
|
75
|
+
f"SKILL.md frontmatter name '{declared}' does not match directory name '{expected}'"
|
|
76
|
+
]
|
|
77
|
+
return []
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def check_references_not_empty(min_nonblank_lines: int = 3) -> list[str]:
|
|
81
|
+
"""Each reference document must contain meaningful prose, not just a title."""
|
|
82
|
+
problems: list[str] = []
|
|
83
|
+
for rel in REFERENCE_FILES:
|
|
84
|
+
path = SKILL_DIR / rel
|
|
85
|
+
if not path.exists():
|
|
86
|
+
continue
|
|
87
|
+
nonblank = [line for line in _read(path).splitlines() if line.strip()]
|
|
88
|
+
if len(nonblank) < min_nonblank_lines:
|
|
89
|
+
problems.append(
|
|
90
|
+
f"{rel} has {len(nonblank)} non-blank line(s); expected at least {min_nonblank_lines}"
|
|
91
|
+
)
|
|
92
|
+
return problems
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def check_evals_records_source() -> list[str]:
|
|
96
|
+
"""evals/evals.json must be valid JSON and record the transcript source."""
|
|
97
|
+
if not EVALS_JSON.exists():
|
|
98
|
+
return ["evals/evals.json is missing"]
|
|
99
|
+
try:
|
|
100
|
+
payload = json.loads(_read(EVALS_JSON))
|
|
101
|
+
except json.JSONDecodeError as exc:
|
|
102
|
+
return [f"evals/evals.json is not valid JSON: {exc}"]
|
|
103
|
+
if not isinstance(payload, dict):
|
|
104
|
+
return ["evals/evals.json must decode to an object"]
|
|
105
|
+
checks = payload.get("checks")
|
|
106
|
+
if not isinstance(checks, list) or not checks:
|
|
107
|
+
return ["evals/evals.json has no `checks` array"]
|
|
108
|
+
names = {check.get("name") for check in checks if isinstance(check, dict)}
|
|
109
|
+
if "transcript-source-recorded" not in names:
|
|
110
|
+
return ["evals/evals.json missing the `transcript-source-recorded` check"]
|
|
111
|
+
return []
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
CHECKS = (
|
|
115
|
+
check_required_files,
|
|
116
|
+
check_skill_name_matches_dir,
|
|
117
|
+
check_references_not_empty,
|
|
118
|
+
check_evals_records_source,
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def main() -> int:
|
|
123
|
+
errors: list[str] = []
|
|
124
|
+
for check in CHECKS:
|
|
125
|
+
errors.extend(check())
|
|
126
|
+
if errors:
|
|
127
|
+
sys.stderr.write("Skill package validation failed:\n")
|
|
128
|
+
for err in errors:
|
|
129
|
+
sys.stderr.write(f" - {err}\n")
|
|
130
|
+
return 1
|
|
131
|
+
print("Skill package structure is valid.")
|
|
132
|
+
return 0
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
if __name__ == "__main__":
|
|
136
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from jinja2 import BaseLoader, ChoiceLoader, Environment, FileSystemLoader, PackageLoader
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _markdown_inline(value: object) -> str:
|
|
10
|
+
"""Keep source metadata inside one Markdown text node."""
|
|
11
|
+
text = " ".join(str(value).splitlines())
|
|
12
|
+
return re.sub(r"([\\`*_\[\]<>()])", r"\\\1", text)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def build_template_env(template_root: Path | None = None) -> Environment:
|
|
16
|
+
loaders: list[BaseLoader] = [PackageLoader("ytk.skill_builder", "templates")]
|
|
17
|
+
if template_root is not None:
|
|
18
|
+
loaders.insert(0, FileSystemLoader(str(template_root)))
|
|
19
|
+
|
|
20
|
+
env = Environment(
|
|
21
|
+
loader=ChoiceLoader(loaders),
|
|
22
|
+
trim_blocks=True,
|
|
23
|
+
lstrip_blocks=True,
|
|
24
|
+
)
|
|
25
|
+
env.filters["code_fence"] = lambda value: "`" * max(
|
|
26
|
+
3, max((len(run) + 1 for run in re.findall(r"`+", str(value))), default=3)
|
|
27
|
+
)
|
|
28
|
+
env.filters["markdown_inline"] = _markdown_inline
|
|
29
|
+
return env
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
TERM_REPLACEMENTS: dict[str, str] = {
|
|
6
|
+
"WPS": "VPS",
|
|
7
|
+
"WpS": "VPS",
|
|
8
|
+
"CloudFair": "Cloudflare",
|
|
9
|
+
"Cloud控": "CloudCone",
|
|
10
|
+
"预明": "域名",
|
|
11
|
+
"平剧": "凭据",
|
|
12
|
+
"天家入站": "入站",
|
|
13
|
+
"叫做VPN": "叫做VPS",
|
|
14
|
+
"密钥就": "密钥",
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
#: Versioned-entity guard (zyr/61b, 2026-10-07 audit): a replacement whose
|
|
18
|
+
#: source term carries a digit-letter version signature (GLM53, NVFP4,
|
|
19
|
+
#: GPT4o) is an entity claim, not ASR noise — silently rewriting it is how
|
|
20
|
+
#: "GLM53 flash" became "GLM-4.5 Flash" in a shipped knowledge doc. Such
|
|
21
|
+
#: entries are skipped; they must be fixed at the evidence layer with an
|
|
22
|
+
#: external source, never in a deterministic noise map.
|
|
23
|
+
_VERSIONED_ENTITY_RE = re.compile(r"[A-Za-z]+\d|\d+[A-Za-z]")
|
|
24
|
+
|
|
25
|
+
# ASCII terms apply at ASCII-word boundaries only — "WPS" must not rewrite
|
|
26
|
+
# the inside of a longer identifier ("WPS2"). NOTE: \b is Unicode-aware and
|
|
27
|
+
# treats CJK neighbors as word characters (叫做WPS had no \b), so the
|
|
28
|
+
# boundary is spelled with explicit ASCII lookarounds.
|
|
29
|
+
_ASCII_TERM_RE = {
|
|
30
|
+
t: re.compile(r"(?<![A-Za-z0-9])" + re.escape(t) + r"(?![A-Za-z0-9])")
|
|
31
|
+
for t in TERM_REPLACEMENTS if t.isascii()
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def normalize_known_term_noise(text: str) -> str:
|
|
36
|
+
normalized = text
|
|
37
|
+
for source, target in TERM_REPLACEMENTS.items():
|
|
38
|
+
if _VERSIONED_ENTITY_RE.search(source) or _VERSIONED_ENTITY_RE.search(target):
|
|
39
|
+
continue # guarded: versioned entities never silently renamed
|
|
40
|
+
if source.isascii():
|
|
41
|
+
normalized = _ASCII_TERM_RE[source].sub(target, normalized)
|
|
42
|
+
else:
|
|
43
|
+
normalized = normalized.replace(source, target)
|
|
44
|
+
return normalized
|