kd-engine 1.0.0__py3-none-any.whl

ytk/models/migrations.py

@@ -0,0 +1,262 @@
+"""T11.1/A26 — strict v1 -> v2 migration.
+
+v1 artifacts never carry ``schema_version``; they are validated by the *old*
+models first and only then converted.  Unknown versions are rejected instead of
+being guessed at, and the original file is never rewritten by this module.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+from typing import Any
+
+from ytk.errors import UserFacingError
+from ytk.models.content import (
+    ContentBlock,
+    ContentBundle,
+    ContentCompleteness,
+    EvidenceRefV2,
+    ExtractionBundleV2,
+    KnowledgeAtomV2,
+    Locator,
+    ParagraphLocator,
+    Processor,
+    TimeRange,
+    TimeRangeLocator,
+    TopicGroup,
+    make_block_id,
+)
+from ytk.models.knowledge import ExtractionBundle
+from ytk.models.source import ContentRef, compute_source_revision
+from ytk.models.transcript import TranscriptBundle
+
+#: v1 transcript source -> v2 evidence processor (provider name is recorded in
+#: ``provider_trace``/metadata instead; evidence type and provider stay separate).
+_V1_PROCESSOR: dict[str, Processor] = {
+    "manual": "subtitle_manual",
+    "auto": "subtitle_auto",
+    "yt_dlp": "subtitle_json3",
+    "asr": "asr",
+    "minimax": "asr",
+}
+
+
+class UnsupportedSchemaVersionError(UserFacingError):
+    """The artifact declares a schema version this build does not understand."""
+
+
+def _digest(payload: object) -> str:
+    return hashlib.sha256(
+        json.dumps(payload, ensure_ascii=False, sort_keys=True, default=str).encode("utf-8")
+    ).hexdigest()
+
+
+def _video_ref(video_id: str, metadata: dict[str, Any]) -> ContentRef:
+    canonical = metadata.get("canonical_url")
+    if not isinstance(canonical, str) or not canonical:
+        canonical = f"https://www.youtube.com/watch?v={video_id}"
+    return ContentRef(platform="youtube", content_id=video_id, canonical_url=canonical)
+
+
+def migrate_transcript_bundle_v1(bundle: TranscriptBundle) -> tuple[ContentRef, ContentBundle]:
+    """Convert a v1 transcript bundle into one v2 video content bundle."""
+    ref = _video_ref(bundle.video_id, bundle.metadata)
+    revision = "v1:" + _digest(
+        {
+            "video_id": bundle.video_id,
+            "transcript_source": bundle.transcript_source,
+            "segments": [segment.model_dump(mode="json") for segment in bundle.segments],
+        }
+    )[:32]
+
+    blocks: list[ContentBlock] = []
+    order = 0
+    for segment in bundle.segments:
+        raw_text = segment.raw_text or segment.text
+        if not raw_text:
+            continue
+        locator = TimeRangeLocator(time=TimeRange(start=segment.start, end=segment.end))
+        blocks.append(
+            ContentBlock(
+                block_id=make_block_id(ref.source_id, locator, raw_text),
+                source_id=ref.source_id,
+                kind="transcript",
+                raw_text=raw_text,
+                cleaned_text=segment.cleaned_text,
+                order=order,
+                locator=locator,
+            )
+        )
+        order += 1
+
+    content = ContentBundle(
+        source=ref,
+        source_revision=revision,
+        blocks=blocks,
+        completeness=ContentCompleteness(status="complete", checked=["transcript_segments"]),
+        metadata={"transcript_source": bundle.transcript_source},
+    )
+    return ref, content
+
+
+def migrate_extraction_bundle_v1(bundle: ExtractionBundle) -> ExtractionBundleV2:
+    """Convert a v1 extraction bundle into a v2 bundle without losing evidence."""
+    ref = _video_ref(bundle.video_id, dict(bundle.metadata))
+    source_id = ref.source_id
+
+    revision = compute_source_revision(
+        source_id,
+        {
+            "schema": 1,
+            "video_id": bundle.video_id,
+            "transcript_source": bundle.transcript_source,
+            "atoms": [atom.model_dump(mode="json") for atom in bundle.knowledge_atoms],
+            "topics": [topic.model_dump(mode="json") for topic in bundle.topic_segments],
+        },
+    )
+
+    blocks: list[ContentBlock] = []
+    evidence_index: dict[str, EvidenceRefV2] = {}
+    # (atom_index, evidence_index_in_atom) -> evidence id
+    atom_evidence_ids: list[list[str]] = []
+    evidence_by_digest: dict[str, str] = {}
+    block_by_digest: dict[str, str] = {}
+    order = 0
+
+    for atom in bundle.knowledge_atoms:
+        ids: list[str] = []
+        for evidence in atom.evidence:
+            digest = _digest([evidence.text, evidence.start, evidence.end])
+            evidence_id = evidence_by_digest.get(digest)
+            if evidence_id is None:
+                locator: Locator = TimeRangeLocator(time=TimeRange(start=evidence.start, end=evidence.end))
+                block_id = make_block_id(source_id, locator, evidence.text)
+                block_digest = _digest([block_id])
+                if block_digest not in block_by_digest:
+                    blocks.append(
+                        ContentBlock(
+                            block_id=block_id,
+                            source_id=source_id,
+                            kind="transcript",
+                            raw_text=evidence.text,
+                            order=order,
+                            locator=locator,
+                        )
+                    )
+                    block_by_digest[block_digest] = block_id
+                    order += 1
+                evidence_id = f"{source_id}@{revision}#ev-{len(evidence_index) + 1:04d}"
+                evidence_index[evidence_id] = EvidenceRefV2(
+                    evidence_id=evidence_id,
+                    source_id=source_id,
+                    source_revision=revision,
+                    block_id=block_id,
+                    quote=evidence.text,
+                    locator=locator,
+                    processor=_V1_PROCESSOR.get(evidence.source, "v1_migration"),
+                )
+                evidence_by_digest[digest] = evidence_id
+            ids.append(evidence_id)
+        atom_evidence_ids.append(ids)
+
+    # Topic segments carry sentence lists but no per-sentence timings.  Attach
+    # the evidence blocks that fall inside the topic window; when a topic has no
+    # timed evidence, keep its text as an explicit paragraph block so no v1
+    # information is dropped.
+    blocks_by_time: list[tuple[float, float, str]] = []
+    for block in blocks:
+        locator = block.locator
+        assert isinstance(locator, TimeRangeLocator)  # migration only creates these
+        blocks_by_time.append((locator.time.start, locator.time.end, block.block_id))
+
+    topics: list[TopicGroup] = []
+    for topic_index, topic in enumerate(bundle.topic_segments):
+        member_ids = [
+            block_id
+            for (start, end, block_id) in blocks_by_time
+            if start < topic.end and end > topic.start
+        ]
+        if not member_ids:
+            paragraph_text = " ".join(sentence for sentence in topic.sentences if sentence).strip()
+            if not paragraph_text:
+                paragraph_text = topic.title
+            locator = ParagraphLocator(paragraph=topic_index)
+            block_id = make_block_id(source_id, locator, paragraph_text)
+            blocks.append(
+                ContentBlock(
+                    block_id=block_id,
+                    source_id=source_id,
+                    kind="paragraph",
+                    raw_text=paragraph_text,
+                    order=order,
+                    locator=locator,
+                )
+            )
+            order += 1
+            member_ids = [block_id]
+        topics.append(
+            TopicGroup(
+                topic_id=topic.id,
+                title=topic.title,
+                block_ids=member_ids,
+                time_range=TimeRange(start=topic.start, end=topic.end),
+                metadata={"embedding_metadata": dict(topic.embedding_metadata)},
+            )
+        )
+
+    knowledge: list[KnowledgeAtomV2] = []
+    for atom_index, atom in enumerate(bundle.knowledge_atoms):
+        knowledge.append(
+            KnowledgeAtomV2(
+                atom_id=atom.id,
+                atom_type=atom.atom_type,
+                content=atom.content,
+                explanation=atom.explanation,
+                source_ids=[source_id],
+                evidence_ids=atom_evidence_ids[atom_index],
+                confidence=atom.confidence,
+                topic_path=list(atom.topic_path),
+                metadata={"timestamps": list(atom.timestamps), "source_video": atom.source_video},
+            )
+        )
+
+    content = ContentBundle(
+        source=ref,
+        source_revision=revision,
+        blocks=blocks,
+        completeness=ContentCompleteness(status="complete", checked=["v1_migration"]),
+        metadata={"transcript_source": bundle.transcript_source},
+    )
+    return ExtractionBundleV2(
+        sources=[ref],
+        contents=[content],
+        evidence_index=evidence_index,
+        topics=topics,
+        knowledge=knowledge,
+        metadata=dict(bundle.metadata),
+        skill_spec=bundle.skill_spec,
+    )
+
+
+def load_bundle_any(path: Path) -> ExtractionBundleV2:
+    """Load a knowledge bundle of any supported schema version.
+
+    ``schema_version == 2`` -> strict v2 validation; missing version -> strict v1
+    validation followed by migration; anything else -> ``UnsupportedSchemaVersionError``.
+    """
+    payload = json.loads(Path(path).read_text(encoding="utf-8"))
+    if not isinstance(payload, dict):
+        raise UnsupportedSchemaVersionError(f"artifact {path} is not a JSON object")
+
+    if "schema_version" in payload:
+        version = payload["schema_version"]
+        if version != 2:
+            raise UnsupportedSchemaVersionError(
+                f"artifact {path} declares unsupported schema_version={version!r}"
+            )
+        return ExtractionBundleV2.model_validate(payload)
+
+    v1 = ExtractionBundle.model_validate(payload)
+    return migrate_extraction_bundle_v1(v1)

ytk/models/skill.py

@@ -0,0 +1,13 @@
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class SkillSpec(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    skill_name: str
+    purpose: str
+    trigger_description: str
+    target_agents: list[str]
+    required_files: list[str]
+    generated_files: list[str]
+    references_map: dict[str, str] = Field(default_factory=dict)

ytk/models/source.py

@@ -0,0 +1,265 @@
+"""T10.1 — unified source identity (platform + canonical content id + variant).
+
+Design rules frozen by the 2026-11-02 implementation plan §4.1:
+
+* ``source_id`` is ``"<platform>:<content_id>"`` plus ``":<variant_id>"`` when the
+  platform has a sub-unit (currently only Bilibili's ``cid``).  The id mixes the
+  platform name in, so the *same numeric id on two platforms can never collide*.
+* ids are strings and keep their case (``BV1GJ411x7h7`` is not ``bv1gj411x7h7``).
+* ``canonical_url`` is the public reference; access tokens (``xsec_token`` …) are
+  stripped from it, and any credential/access data lives in :class:`AccessContext`
+  which is never serialized into artifacts or logs.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import Callable, Mapping
+from typing import Literal
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator, model_validator
+
+from ytk.ingestion.url_parser import ParsedYouTubeUrl
+
+Platform = Literal["youtube", "tiktok", "douyin", "bilibili", "xiaohongshu", "local"]
+ContentType = Literal["video", "image_note"]
+
+SUPPORTED_PLATFORMS: tuple[str, ...] = ("youtube", "tiktok", "douyin", "bilibili", "xiaohongshu")
+SUPPORTED_CONTENT_TYPES: tuple[str, ...] = ("video", "image_note")
+
+#: Query parameters that carry access rights or tracking; never public.
+SENSITIVE_QUERY_PARAMS: frozenset[str] = frozenset(
+    {
+        "xsec_token",
+        "xsec_source",
+        "xsec_app_secuboid",
+        "mstoken",
+        "share_token",
+        "share_source",
+        "share_session_id",
+        "access_key",
+        "sessdata",
+        "bili_jct",
+        "dedeuserid",
+        "dedeuserid__ckmd5",
+        "vd_source",
+        "verifyfp",
+        "s_v_web_id",
+        "x-bogus",
+        "x_bogus",
+        "_signature",
+        "ttwid",
+        "appuid",
+        "apptime",
+        "shareredid",
+        "sign",
+        "signature",
+        "expires",
+        "token",
+        "access_token",
+        "sessionid",
+        "sid",
+        "cookie",
+        "aci",
+        "buvid",
+        "w_rid",
+        "wts",
+        "auth_key",
+        "sig",
+        "key",
+        "expire",
+        "a_bogus",
+        "x-amz-signature",
+        "x-amz-credential",
+        "x-amz-security-token",
+        "mstoken_v2",
+        "expiration",
+        "refresh_token",
+        "session_id",
+        "authkey",
+        "hdnts",
+        "policy",
+        "playauth",
+        "nsec_token",
+        "appkey",
+        "api_key",
+        "apikey",
+        "id_token",
+    }
+)
+
+
+def sanitize_public_url(url: str) -> str:
+    """Return *url* with access/tracking parameters and userinfo removed.
+
+    The renderers and knowledge artifacts only ever get this form.  Raises
+    ``ValueError`` for anything that is not an absolute http(s) URL.
+    """
+    if not isinstance(url, str) or not url.strip():
+        raise ValueError("url must be a non-empty string")
+    parts = urlsplit(url.strip())
+    if parts.scheme not in {"http", "https"}:
+        raise ValueError(f"unsupported URL scheme: {parts.scheme or '<missing>'}")
+    if not parts.hostname:
+        raise ValueError("URL is missing a host")
+    if parts.username or parts.password:
+        raise ValueError("URL must not embed credentials")
+    host = parts.hostname.lower()
+    netloc = host if parts.port in (None, 80, 443) else f"{host}:{parts.port}"
+    kept = [
+        (name, value)
+        for name, value in parse_qsl(parts.query, keep_blank_values=True)
+        if name.casefold() not in SENSITIVE_QUERY_PARAMS
+    ]
+    path = parts.path or "/"
+    return urlunsplit((parts.scheme, netloc, path, urlencode(kept), ""))
+
+
+class AccessContext(BaseModel):
+    """Credentials/proxy/header context for one acquisition task.
+
+    Deliberately frozen and never embedded in ``ContentRef`` / ``ContentBundle`` /
+    renderer output.  ``redacted()`` is the only shape allowed in logs/diagnostics.
+    """
+
+    model_config = ConfigDict(extra="forbid", frozen=True, hide_input_in_errors=True)
+
+    cookies_file: str | None = Field(default=None, exclude=True, repr=False)
+    cookies_from_browser: str | None = Field(default=None, exclude=True, repr=False)
+    proxy: str | None = Field(default=None, exclude=True, repr=False)
+    headers: Mapping[str, str] = Field(default_factory=dict, exclude=True, repr=False)
+    tokens: Mapping[str, str] = Field(default_factory=dict, exclude=True, repr=False)
+    resolved_url_callback: Callable[[str], None] | None = Field(default=None, exclude=True, repr=False)
+
+    def redacted(self) -> dict[str, object]:
+        """Credential-free summary safe for logs, manifests and reports."""
+        proxy_host = None
+        if self.proxy:
+            proxy_host = urlsplit(self.proxy).hostname or "<invalid>"
+        return {
+            "cookies_file": "<configured>" if self.cookies_file else None,
+            "cookies_from_browser": "<configured>" if self.cookies_from_browser else None,
+            "proxy_host": proxy_host,
+            "header_names": sorted(self.headers),
+            "token_names": sorted(self.tokens),
+        }
+
+
+class ContentRef(BaseModel):
+    """Stable identity of one content unit (a video, or an image-note)."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    platform: Platform
+    content_id: str = Field(min_length=1)
+    variant_id: str | None = None
+    content_type: ContentType = "video"
+    canonical_url: str
+    alias_urls: list[str] = Field(default_factory=list)
+    display: Mapping[str, str] = Field(default_factory=dict)
+
+    @model_validator(mode="before")
+    @classmethod
+    def _validate_public_source_id(cls, value: object) -> object:
+        if not isinstance(value, Mapping) or "source_id" not in value:
+            return value
+        fields = dict(value)
+        supplied = fields.pop("source_id")
+        expected = f"{fields.get('platform')}:{fields.get('content_id')}"
+        if fields.get("variant_id"):
+            expected += f":{fields['variant_id']}"
+        if supplied != expected:
+            raise ValueError("source_id does not match platform/content_id/variant_id")
+        return fields
+
+    @field_validator("canonical_url")
+    @classmethod
+    def _validate_canonical_url(cls, value: str, info: ValidationInfo) -> str:
+        if info.data.get("platform") == "local":
+            parts = urlsplit(value)
+            if parts.scheme != "file" or parts.netloc or not parts.path.startswith("/") or parts.query or parts.fragment:
+                raise ValueError("local sources require an absolute local file URI")
+            return value
+        return sanitize_public_url(value)
+
+    @field_validator("alias_urls")
+    @classmethod
+    def _validate_alias_urls(cls, values: list[str]) -> list[str]:
+        sanitized: list[str] = []
+        for value in values:
+            cleaned = sanitize_public_url(value)
+            if cleaned not in sanitized:
+                sanitized.append(cleaned)
+        return sanitized
+
+    @field_validator("content_id", "variant_id")
+    @classmethod
+    def _validate_identity_component(cls, value: str | None) -> str | None:
+        if value is not None and (not value or any(char.isspace() for char in value) or ":" in value):
+            raise ValueError("source identity components must be non-empty and contain no whitespace or colon")
+        return value
+
+    @model_validator(mode="after")
+    def _validate_variant(self) -> ContentRef:
+        if self.variant_id is not None and not self.variant_id.strip():
+            raise ValueError("variant_id must be non-empty when present")
+        return self
+
+    @property
+    def source_id(self) -> str:
+        base = f"{self.platform}:{self.content_id}"
+        return f"{base}:{self.variant_id}" if self.variant_id else base
+
+    def with_variant(self, variant_id: str, canonical_url: str | None = None) -> ContentRef:
+        """Return a copy bound to a platform sub-unit (Bilibili ``cid``).
+
+        Revalidate the complete reference: ``model_copy(update=...)`` skips both
+        identity checks and public URL sanitization.
+        """
+        update = self.model_dump()
+        update["variant_id"] = variant_id
+        if canonical_url is not None:
+            update["canonical_url"] = canonical_url
+        return type(self).model_validate(update)
+
+    def public_dict(self) -> dict[str, object]:
+        """JSON-safe public shape (no access parameters by construction)."""
+        return {
+            "platform": self.platform,
+            "content_id": self.content_id,
+            "variant_id": self.variant_id,
+            "content_type": self.content_type,
+            "source_id": self.source_id,
+            "canonical_url": self.canonical_url,
+            "alias_urls": list(self.alias_urls),
+            "display": dict(self.display),
+        }
+
+
+def from_youtube_parsed(parsed: ParsedYouTubeUrl) -> ContentRef:
+    """v1 compatibility: turn the legacy parsed YouTube URL into a ContentRef."""
+    return ContentRef(
+        platform="youtube",
+        content_id=parsed.video_id,
+        canonical_url=parsed.canonical_url,
+        alias_urls=[] if parsed.raw_url == parsed.canonical_url else [parsed.raw_url],
+    )
+
+
+def compute_source_revision(source_id: str, fingerprint_parts: Mapping[str, object]) -> str:
+    """Deterministic source revision from the *acquired evidence* fingerprint.
+
+    Only facts that change the delivered content may feed this hash (subtitle
+    language/format, media digest, note text digest …).  Renderer settings and
+    prompt/model versions must NOT be included: changing a template must not
+    invalidate acquired content.
+    """
+    payload = json.dumps(
+        {"source_id": source_id, "parts": {key: fingerprint_parts[key] for key in sorted(fingerprint_parts)}},
+        ensure_ascii=False,
+        sort_keys=True,
+        separators=(",", ":"),
+    )
+    return "sha256:" + hashlib.sha256(payload.encode("utf-8")).hexdigest()

ytk/models/topic.py

@@ -0,0 +1,18 @@
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+
+class TopicSegment(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    id: str
+    title: str
+    start: float = Field(ge=0)
+    end: float = Field(ge=0)
+    sentences: list[str]
+    embedding_metadata: dict[str, object] = Field(default_factory=dict)
+
+    @model_validator(mode="after")
+    def validate_time_range(self) -> "TopicSegment":
+        if self.end < self.start:
+            raise ValueError("end must be greater than or equal to start")
+        return self

ytk/models/transcript.py

@@ -0,0 +1,30 @@
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from ytk.models.common import TranscriptSource
+
+
+class TranscriptSegment(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    text: str
+    start: float = Field(ge=0)
+    end: float = Field(ge=0)
+    source: TranscriptSource
+    raw_text: str
+    cleaned_text: str | None = None
+
+    @model_validator(mode="after")
+    def validate_time_range(self) -> "TranscriptSegment":
+        if self.end < self.start:
+            raise ValueError("end must be greater than or equal to start")
+        return self
+
+
+class TranscriptBundle(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    video_id: str
+    canonical_url: str
+    transcript_source: TranscriptSource
+    segments: list[TranscriptSegment]
+    metadata: dict[str, object]

ytk/ocr/__init__.py

@@ -0,0 +1,65 @@
+"""YTK OCR: local-only image/video-frame recognition (RapidOCR) with location evidence.
+
+Public surface:
+
+* :mod:`ytk.ocr.engine` — ``OCRLine``, ``OCREngine`` protocol, ``NullOCR`` and the
+  lazy :func:`get_engine` factory (raises ``ModelUnavailableError`` when the
+  optional engine/models are not provisioned; never downloads at inference time).
+* :mod:`ytk.ocr.rapidocr_engine` — the RapidOCR implementation.
+* :mod:`ytk.ocr.layout` — pixel-budget long-image tiling, original-coordinate
+  bbox mapping, overlap de-duplication and reading-order sorting.
+* :mod:`ytk.ocr.pipeline` — ``run_ocr_for_content`` (image notes) and
+  ``run_ocr_for_frames`` (T32.2 budget-bound video frames).
+"""
+
+from __future__ import annotations
+
+from ytk.ocr.engine import (
+    DEFAULT_MODEL_DIR,
+    DEFAULT_MODEL_FILENAMES,
+    ENV_OCR_MODEL_DIR,
+    ModelPaths,
+    NullOCR,
+    OCREngine,
+    OCRLine,
+    engine_available,
+    get_engine,
+    resolve_model_paths,
+)
+from ytk.ocr.layout import (
+    LongImageResult,
+    Tile,
+    TileBudget,
+    dedupe_lines,
+    normalize_text,
+    ocr_long_image,
+    plan_tiles,
+    reading_order,
+    text_similarity,
+)
+from ytk.ocr.pipeline import FrameOCRBudget, run_ocr_for_content, run_ocr_for_frames
+
+__all__ = [
+    "DEFAULT_MODEL_DIR",
+    "DEFAULT_MODEL_FILENAMES",
+    "ENV_OCR_MODEL_DIR",
+    "FrameOCRBudget",
+    "LongImageResult",
+    "ModelPaths",
+    "NullOCR",
+    "OCREngine",
+    "OCRLine",
+    "Tile",
+    "TileBudget",
+    "dedupe_lines",
+    "engine_available",
+    "get_engine",
+    "normalize_text",
+    "ocr_long_image",
+    "plan_tiles",
+    "reading_order",
+    "resolve_model_paths",
+    "run_ocr_for_content",
+    "run_ocr_for_frames",
+    "text_similarity",
+]