kd-engine 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. kd_engine-1.0.0.dist-info/METADATA +19 -0
  2. kd_engine-1.0.0.dist-info/RECORD +118 -0
  3. kd_engine-1.0.0.dist-info/WHEEL +5 -0
  4. kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
  5. kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
  6. kd_engine-1.0.0.dist-info/top_level.txt +1 -0
  7. ytk/__init__.py +2 -0
  8. ytk/__main__.py +21 -0
  9. ytk/batch/__init__.py +39 -0
  10. ytk/batch/runner.py +1149 -0
  11. ytk/cache/__init__.py +34 -0
  12. ytk/cache/v2.py +562 -0
  13. ytk/cleaning/density.py +40 -0
  14. ytk/cleaning/fillers.py +18 -0
  15. ytk/cleaning/sentence_boundary.py +30 -0
  16. ytk/cleaning/service.py +62 -0
  17. ytk/cli.py +1281 -0
  18. ytk/commands/__init__.py +3 -0
  19. ytk/commands/merge.py +600 -0
  20. ytk/config.py +757 -0
  21. ytk/dedup/service.py +166 -0
  22. ytk/doctor.py +891 -0
  23. ytk/dotenv.py +334 -0
  24. ytk/errors.py +38 -0
  25. ytk/extraction/service.py +451 -0
  26. ytk/extraction/strategies/base.py +16 -0
  27. ytk/extraction/strategies/code_block.py +285 -0
  28. ytk/extraction/strategies/heuristic.py +791 -0
  29. ytk/extraction/strategies/local_model.py +56 -0
  30. ytk/extraction/text_context.py +177 -0
  31. ytk/extraction/visual_context.py +159 -0
  32. ytk/ingestion/adapters/__init__.py +1 -0
  33. ytk/ingestion/adapters/_common.py +865 -0
  34. ytk/ingestion/adapters/base.py +160 -0
  35. ytk/ingestion/adapters/bilibili.py +271 -0
  36. ytk/ingestion/adapters/douyin.py +445 -0
  37. ytk/ingestion/adapters/tiktok.py +102 -0
  38. ytk/ingestion/adapters/xiaohongshu.py +758 -0
  39. ytk/ingestion/adapters/youtube.py +74 -0
  40. ytk/ingestion/browser_acquired.py +220 -0
  41. ytk/ingestion/http_client.py +43 -0
  42. ytk/ingestion/local_import.py +202 -0
  43. ytk/ingestion/media.py +468 -0
  44. ytk/ingestion/providers/asr.py +902 -0
  45. ytk/ingestion/providers/base.py +13 -0
  46. ytk/ingestion/providers/dtk_client.py +816 -0
  47. ytk/ingestion/providers/minimax.py +350 -0
  48. ytk/ingestion/providers/transcript_api.py +336 -0
  49. ytk/ingestion/providers/xhs_client.py +646 -0
  50. ytk/ingestion/providers/ytdlp.py +400 -0
  51. ytk/ingestion/registry.py +189 -0
  52. ytk/ingestion/resolver.py +417 -0
  53. ytk/ingestion/search.py +56 -0
  54. ytk/ingestion/service.py +167 -0
  55. ytk/ingestion/subtitles.py +602 -0
  56. ytk/ingestion/transcript_cache.py +212 -0
  57. ytk/ingestion/url_parser.py +53 -0
  58. ytk/ingestion/video_frames.py +292 -0
  59. ytk/ingestion/ytdlp_acquire.py +754 -0
  60. ytk/logging.py +383 -0
  61. ytk/models/common.py +20 -0
  62. ytk/models/content.py +451 -0
  63. ytk/models/knowledge.py +43 -0
  64. ytk/models/migrations.py +262 -0
  65. ytk/models/skill.py +13 -0
  66. ytk/models/source.py +265 -0
  67. ytk/models/topic.py +18 -0
  68. ytk/models/transcript.py +30 -0
  69. ytk/ocr/__init__.py +65 -0
  70. ytk/ocr/engine.py +245 -0
  71. ytk/ocr/layout.py +347 -0
  72. ytk/ocr/pipeline.py +440 -0
  73. ytk/ocr/rapidocr_engine.py +144 -0
  74. ytk/pipeline/acquisition_checkpoint.py +240 -0
  75. ytk/pipeline/artifacts.py +18 -0
  76. ytk/pipeline/orchestrator.py +598 -0
  77. ytk/pipeline/processing_identity.py +18 -0
  78. ytk/pipeline/retry.py +728 -0
  79. ytk/pipeline/runners.py +578 -0
  80. ytk/pipeline/state.py +1107 -0
  81. ytk/pipeline/v2.py +1863 -0
  82. ytk/py.typed +1 -0
  83. ytk/quality/__init__.py +9 -0
  84. ytk/quality/completeness.py +260 -0
  85. ytk/renderers/base.py +9 -0
  86. ytk/renderers/citation_alignment.py +617 -0
  87. ytk/renderers/evidence_formatter.py +362 -0
  88. ytk/renderers/json_renderer.py +13 -0
  89. ytk/renderers/jsonl_renderer.py +18 -0
  90. ytk/renderers/knowledge_doc_renderer.py +1984 -0
  91. ytk/renderers/markdown_renderer.py +540 -0
  92. ytk/renderers/obsidian_renderer.py +138 -0
  93. ytk/renderers/skill_package.py +21 -0
  94. ytk/renderers/source_links.py +309 -0
  95. ytk/renderers/summary_renderer.py +163 -0
  96. ytk/renderers/v2.py +171 -0
  97. ytk/renderers/v2_jsonl.py +165 -0
  98. ytk/renderers/v2_markdown.py +477 -0
  99. ytk/renderers/v2_obsidian.py +113 -0
  100. ytk/renderers/v2_summary.py +220 -0
  101. ytk/segmentation/change_points.py +50 -0
  102. ytk/segmentation/embeddings.py +45 -0
  103. ytk/segmentation/service.py +318 -0
  104. ytk/skill_builder/__init__.py +1 -0
  105. ytk/skill_builder/builder.py +419 -0
  106. ytk/skill_builder/templates/SKILL.md.j2 +85 -0
  107. ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
  108. ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
  109. ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
  110. ytk/skill_builder/templates/references/commands.md.j2 +34 -0
  111. ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
  112. ytk/skill_builder/templates/references/configs.md.j2 +31 -0
  113. ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
  114. ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
  115. ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
  116. ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
  117. ytk/skill_builder/templates.py +29 -0
  118. ytk/text_normalization.py +44 -0
@@ -0,0 +1,14 @@
1
+ from pathlib import Path
2
+
3
+
4
+ def main() -> None:
5
+ target = Path(".")
6
+ print(
7
+ "This package was generated from extraction artifacts. "
8
+ "Rebuild it from source data by calling SkillBuilder with the original bundle."
9
+ )
10
+ print(f"Package root: {target.resolve()}")
11
+
12
+
13
+ if __name__ == "__main__":
14
+ main()
@@ -0,0 +1,136 @@
1
+ #!/usr/bin/env python3
2
+ """Validate the structure of a generated skill package.
3
+
4
+ Run from inside the skill directory (the parent of ``scripts/``):
5
+
6
+ python3 scripts/validate_output.py
7
+
8
+ The checks mirror the ``evals/evals.json`` contract emitted alongside this
9
+ script, so a generated package can self-verify before another agent consumes
10
+ it. Keep the two in sync if you add new structural rules.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import json
16
+ import re
17
+ import sys
18
+ from pathlib import Path
19
+
20
+
21
+ SKILL_DIR = Path(__file__).resolve().parent.parent
22
+ SKILL_MD = SKILL_DIR / "SKILL.md"
23
+ EVALS_JSON = SKILL_DIR / "evals" / "evals.json"
24
+
25
+ REQUIRED_FILES = (
26
+ "SKILL.md",
27
+ "references/source-summary.md",
28
+ "references/commands.md",
29
+ "references/configs.md",
30
+ "references/concepts.md",
31
+ "references/evidence-policy.md",
32
+ "examples/example-input.md",
33
+ "examples/example-output.md",
34
+ "evals/evals.json",
35
+ "scripts/build_skill.py",
36
+ "scripts/validate_output.py",
37
+ )
38
+
39
+ REFERENCE_FILES = (
40
+ "references/source-summary.md",
41
+ "references/commands.md",
42
+ "references/configs.md",
43
+ "references/concepts.md",
44
+ "references/evidence-policy.md",
45
+ )
46
+
47
+ # Matches a YAML frontmatter name field. The skill-creator linter enforces
48
+ # that this matches the directory name, so we mirror that rule locally.
49
+ _FRONTMATTER_NAME_RE = re.compile(r"^---\s*\nname:\s*(\S+)\s*$", re.MULTILINE)
50
+
51
+
52
+ def _read(path: Path) -> str:
53
+ if not path.exists():
54
+ return ""
55
+ return path.read_text(encoding="utf-8")
56
+
57
+
58
+ def check_required_files() -> list[str]:
59
+ """All hand-authored and rendered files must be present."""
60
+ missing = [name for name in REQUIRED_FILES if not (SKILL_DIR / name).exists()]
61
+ return [f"missing required file: {name}" for name in missing]
62
+
63
+
64
+ def check_skill_name_matches_dir() -> list[str]:
65
+ """SKILL.md frontmatter name must equal the directory name."""
66
+ if not SKILL_MD.exists():
67
+ return []
68
+ match = _FRONTMATTER_NAME_RE.search(_read(SKILL_MD))
69
+ if not match:
70
+ return ["SKILL.md frontmatter is missing the `name` field"]
71
+ declared = match.group(1)
72
+ expected = SKILL_DIR.name
73
+ if declared != expected:
74
+ return [
75
+ f"SKILL.md frontmatter name '{declared}' does not match directory name '{expected}'"
76
+ ]
77
+ return []
78
+
79
+
80
+ def check_references_not_empty(min_nonblank_lines: int = 3) -> list[str]:
81
+ """Each reference document must contain meaningful prose, not just a title."""
82
+ problems: list[str] = []
83
+ for rel in REFERENCE_FILES:
84
+ path = SKILL_DIR / rel
85
+ if not path.exists():
86
+ continue
87
+ nonblank = [line for line in _read(path).splitlines() if line.strip()]
88
+ if len(nonblank) < min_nonblank_lines:
89
+ problems.append(
90
+ f"{rel} has {len(nonblank)} non-blank line(s); expected at least {min_nonblank_lines}"
91
+ )
92
+ return problems
93
+
94
+
95
+ def check_evals_records_source() -> list[str]:
96
+ """evals/evals.json must be valid JSON and record the transcript source."""
97
+ if not EVALS_JSON.exists():
98
+ return ["evals/evals.json is missing"]
99
+ try:
100
+ payload = json.loads(_read(EVALS_JSON))
101
+ except json.JSONDecodeError as exc:
102
+ return [f"evals/evals.json is not valid JSON: {exc}"]
103
+ if not isinstance(payload, dict):
104
+ return ["evals/evals.json must decode to an object"]
105
+ checks = payload.get("checks")
106
+ if not isinstance(checks, list) or not checks:
107
+ return ["evals/evals.json has no `checks` array"]
108
+ names = {check.get("name") for check in checks if isinstance(check, dict)}
109
+ if "transcript-source-recorded" not in names:
110
+ return ["evals/evals.json missing the `transcript-source-recorded` check"]
111
+ return []
112
+
113
+
114
+ CHECKS = (
115
+ check_required_files,
116
+ check_skill_name_matches_dir,
117
+ check_references_not_empty,
118
+ check_evals_records_source,
119
+ )
120
+
121
+
122
+ def main() -> int:
123
+ errors: list[str] = []
124
+ for check in CHECKS:
125
+ errors.extend(check())
126
+ if errors:
127
+ sys.stderr.write("Skill package validation failed:\n")
128
+ for err in errors:
129
+ sys.stderr.write(f" - {err}\n")
130
+ return 1
131
+ print("Skill package structure is valid.")
132
+ return 0
133
+
134
+
135
+ if __name__ == "__main__":
136
+ raise SystemExit(main())
@@ -0,0 +1,29 @@
1
+ from __future__ import annotations
2
+
3
+ import re
4
+ from pathlib import Path
5
+
6
+ from jinja2 import BaseLoader, ChoiceLoader, Environment, FileSystemLoader, PackageLoader
7
+
8
+
9
+ def _markdown_inline(value: object) -> str:
10
+ """Keep source metadata inside one Markdown text node."""
11
+ text = " ".join(str(value).splitlines())
12
+ return re.sub(r"([\\`*_\[\]<>()])", r"\\\1", text)
13
+
14
+
15
+ def build_template_env(template_root: Path | None = None) -> Environment:
16
+ loaders: list[BaseLoader] = [PackageLoader("ytk.skill_builder", "templates")]
17
+ if template_root is not None:
18
+ loaders.insert(0, FileSystemLoader(str(template_root)))
19
+
20
+ env = Environment(
21
+ loader=ChoiceLoader(loaders),
22
+ trim_blocks=True,
23
+ lstrip_blocks=True,
24
+ )
25
+ env.filters["code_fence"] = lambda value: "`" * max(
26
+ 3, max((len(run) + 1 for run in re.findall(r"`+", str(value))), default=3)
27
+ )
28
+ env.filters["markdown_inline"] = _markdown_inline
29
+ return env
@@ -0,0 +1,44 @@
1
+ from __future__ import annotations
2
+
3
+ import re
4
+
5
+ TERM_REPLACEMENTS: dict[str, str] = {
6
+ "WPS": "VPS",
7
+ "WpS": "VPS",
8
+ "CloudFair": "Cloudflare",
9
+ "Cloud控": "CloudCone",
10
+ "预明": "域名",
11
+ "平剧": "凭据",
12
+ "天家入站": "入站",
13
+ "叫做VPN": "叫做VPS",
14
+ "密钥就": "密钥",
15
+ }
16
+
17
+ #: Versioned-entity guard (zyr/61b, 2026-10-07 audit): a replacement whose
18
+ #: source term carries a digit-letter version signature (GLM53, NVFP4,
19
+ #: GPT4o) is an entity claim, not ASR noise — silently rewriting it is how
20
+ #: "GLM53 flash" became "GLM-4.5 Flash" in a shipped knowledge doc. Such
21
+ #: entries are skipped; they must be fixed at the evidence layer with an
22
+ #: external source, never in a deterministic noise map.
23
+ _VERSIONED_ENTITY_RE = re.compile(r"[A-Za-z]+\d|\d+[A-Za-z]")
24
+
25
+ # ASCII terms apply at ASCII-word boundaries only — "WPS" must not rewrite
26
+ # the inside of a longer identifier ("WPS2"). NOTE: \b is Unicode-aware and
27
+ # treats CJK neighbors as word characters (叫做WPS had no \b), so the
28
+ # boundary is spelled with explicit ASCII lookarounds.
29
+ _ASCII_TERM_RE = {
30
+ t: re.compile(r"(?<![A-Za-z0-9])" + re.escape(t) + r"(?![A-Za-z0-9])")
31
+ for t in TERM_REPLACEMENTS if t.isascii()
32
+ }
33
+
34
+
35
+ def normalize_known_term_noise(text: str) -> str:
36
+ normalized = text
37
+ for source, target in TERM_REPLACEMENTS.items():
38
+ if _VERSIONED_ENTITY_RE.search(source) or _VERSIONED_ENTITY_RE.search(target):
39
+ continue # guarded: versioned entities never silently renamed
40
+ if source.isascii():
41
+ normalized = _ASCII_TERM_RE[source].sub(target, normalized)
42
+ else:
43
+ normalized = normalized.replace(source, target)
44
+ return normalized