kd-engine 1.0.0__py3-none-any.whl

ytk/models/content.py

@@ -0,0 +1,451 @@
+"""T11.1 — v2 content + evidence model (video **and** image-note coordinates).
+
+Frozen contract (plan §4.3):
+
+* four locator kinds — ``time_range``, ``paragraph``, ``image_region``,
+  ``video_frame`` — discriminated by ``kind``; image indices are 1-based for
+  humans, bboxes are original-image pixel ``xyxy`` with the source dimensions
+  and a coordinate-system version recorded;
+* ``block_id`` is stable **within one source revision** and derived from
+  (source, locator, raw text) — processing settings (cleaner, model, prompt)
+  can change without re-keying evidence;
+* raw text and cleaned text are both stored; cleaning never overwrites raw;
+* ``image_note`` content may not carry invented timestamps (no ``[00:00]``);
+* ``ExtractionBundleV2`` validates that every evidence reference resolves to a
+  block of the *same* source revision, and that every atom/topic reference
+  resolves.  Nothing is "repair-by-guess".
+"""
+
+from __future__ import annotations
+
+import hashlib
+from pathlib import PurePosixPath, PureWindowsPath
+from typing import Annotated, Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator
+
+from ytk.models.skill import SkillSpec
+from ytk.models.source import ContentRef
+
+BBox = tuple[float, float, float, float]
+CoordinateVersion = Literal["px-xyxy-v1"]
+BlockKind = Literal["transcript", "caption", "ocr", "paragraph", "title", "description"]
+AtomType = Literal[
+    "terminal_command",
+    "configuration_snippet",
+    "best_practice",
+    "core_concept",
+    "workflow_step",
+    "code_block",
+    "command_sequence",
+    "insight",
+    "reference",
+    # 2026-11-04: 粗粒度分段（显式低置信）。只在用户显式开启 coarse 模式时产生，
+    # 用于"内容完整但抽不出结构化知识"的源，避免把 NO_KNOWLEDGE 变成空台账。
+    "segment_summary",
+]
+Processor = Literal[
+    "subtitle_manual",
+    "subtitle_auto",
+    "subtitle_json3",
+    "subtitle_srt",
+    "subtitle_vtt",
+    "asr",
+    "ocr",
+    "author_text",
+    "video_frame_ocr",
+    "user_provided_subtitle",
+    "visual_review",
+    "v1_migration",
+]
+
+
+class TimeRange(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    start: float = Field(ge=0, allow_inf_nan=False)
+    end: float = Field(ge=0, allow_inf_nan=False)
+
+    @model_validator(mode="after")
+    def _validate_order(self) -> TimeRange:
+        if self.end < self.start:
+            raise ValueError("end must be greater than or equal to start")
+        return self
+
+
+class TimeRangeLocator(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    kind: Literal["time_range"] = "time_range"
+    time: TimeRange
+
+
+class ParagraphLocator(BaseModel):
+    """Body-paragraph position (image notes, author text)."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    kind: Literal["paragraph"] = "paragraph"
+    paragraph: int = Field(ge=0)
+
+
+class ImageRegionLocator(BaseModel):
+    """A region of one image, in original-image pixel coordinates."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    kind: Literal["image_region"] = "image_region"
+    image_index: int = Field(ge=1, description="1-based image order shown to users")
+    image_width: int = Field(gt=0)
+    image_height: int = Field(gt=0)
+    bbox: BBox | None = None
+    coordinate_version: CoordinateVersion = "px-xyxy-v1"
+    tile_offset: tuple[int, int] = (0, 0)
+
+    @model_validator(mode="after")
+    def _validate_bbox(self) -> ImageRegionLocator:
+        if self.bbox is None:
+            return self
+        x0, y0, x1, y1 = self.bbox
+        if not (0 <= x0 < x1 <= self.image_width and 0 <= y0 < y1 <= self.image_height):
+            raise ValueError(
+                "bbox must be a positive-area xyxy rectangle inside the original image "
+                f"({self.image_width}x{self.image_height})"
+            )
+        return self
+
+
+class VideoFrameLocator(BaseModel):
+    """A sampled video frame (画面 OCR is independent evidence)."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    kind: Literal["video_frame"] = "video_frame"
+    time: float = Field(ge=0, allow_inf_nan=False)
+    image_width: int | None = Field(default=None, gt=0)
+    image_height: int | None = Field(default=None, gt=0)
+    bbox: BBox | None = None
+    coordinate_version: CoordinateVersion = "px-xyxy-v1"
+
+    @model_validator(mode="after")
+    def _validate_bbox(self) -> VideoFrameLocator:
+        if self.bbox is None:
+            return self
+        if self.image_width is None or self.image_height is None:
+            raise ValueError("video_frame bbox requires image_width and image_height")
+        x0, y0, x1, y1 = self.bbox
+        if not (0 <= x0 < x1 <= self.image_width and 0 <= y0 < y1 <= self.image_height):
+            raise ValueError("bbox must be a positive-area xyxy rectangle inside the frame")
+        return self
+
+
+Locator = Annotated[
+    TimeRangeLocator | ParagraphLocator | ImageRegionLocator | VideoFrameLocator,
+    Field(discriminator="kind"),
+]
+
+
+def locator_key(locator: TimeRangeLocator | ParagraphLocator | ImageRegionLocator | VideoFrameLocator) -> str:
+    """Canonical, order-independent key for block-identity hashing."""
+    return locator.model_dump_json(exclude_none=True)
+
+
+def make_block_id(
+    source_id: str,
+    locator: TimeRangeLocator | ParagraphLocator | ImageRegionLocator | VideoFrameLocator,
+    raw_text: str,
+) -> str:
+    """Stable block id: source + locator + raw text, never the cleaned form."""
+    payload = f"{source_id}\x1f{locator_key(locator)}\x1f{raw_text}"
+    return "blk-" + hashlib.sha256(payload.encode("utf-8")).hexdigest()[:20]
+
+
+class ContentBlock(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    block_id: str = Field(min_length=1)
+    source_id: str = Field(min_length=1)
+    kind: BlockKind
+    raw_text: str = Field(min_length=1)
+    cleaned_text: str | None = None
+    language: str | None = None
+    order: int = Field(ge=0)
+    locator: Locator
+
+
+class MediaAsset(BaseModel):
+    """A local media file that belongs to a source (portable relative path)."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    asset_id: str = Field(min_length=1)
+    source_id: str = Field(min_length=1)
+    relative_path: str = Field(min_length=1)
+    media_type: Literal["video", "audio", "image", "frame"]
+    order: int = Field(ge=0)
+    byte_size: int = Field(ge=0)
+    sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
+
+    @field_validator("relative_path")
+    @classmethod
+    def _validate_relative_path(cls, value: str) -> str:
+        path = PurePosixPath(value)
+        if (
+            path.is_absolute() or PureWindowsPath(value).drive or "\\" in value
+            or ".." in path.parts or not path.parts or "\x00" in value
+        ):
+            raise ValueError("relative_path must stay inside the portable artifact package")
+        return value
+
+
+class ContentCompleteness(BaseModel):
+    """Whether the acquired content is provably complete.
+
+    ``unknown`` NEVER converts to ``complete`` downstream; it maps to a
+    recoverable BLOCKED with a reason code.
+    """
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    status: Literal["complete", "partial", "unknown"]
+    checked: list[str] = Field(default_factory=list)
+    missing: list[str] = Field(default_factory=list)
+    reason_code: str | None = None
+
+    @model_validator(mode="after")
+    def _validate_status(self) -> ContentCompleteness:
+        if self.status == "complete" and self.missing:
+            raise ValueError("complete content cannot report missing items")
+        return self
+
+
+class ProviderTrace(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    provider: str = Field(min_length=1)
+    provider_version: str | None = None
+    stage: Literal["resolve", "acquire", "subtitle", "media", "asr", "ocr"]
+    status: Literal["ok", "failed", "skipped", "unknown"]
+    notes: str | None = None
+
+
+class ContentBundle(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    schema_version: Literal[2] = 2
+    source: ContentRef
+    source_revision: str = Field(min_length=1)
+    assets: list[MediaAsset] = Field(default_factory=list)
+    blocks: list[ContentBlock] = Field(default_factory=list)
+    completeness: ContentCompleteness
+    provider_trace: list[ProviderTrace] = Field(default_factory=list)
+    metadata: dict[str, object] = Field(default_factory=dict)
+
+    @model_validator(mode="after")
+    def _validate_bundle(self) -> ContentBundle:
+        source_id = self.source.source_id
+        seen: set[str] = set()
+        for block in self.blocks:
+            if block.block_id in seen:
+                raise ValueError(f"duplicate block id: {block.block_id}")
+            seen.add(block.block_id)
+            if block.source_id != source_id:
+                raise ValueError(
+                    f"block {block.block_id} belongs to source {block.source_id!r}, "
+                    f"bundle is {source_id!r}"
+                )
+        for asset in self.assets:
+            if asset.source_id != source_id:
+                raise ValueError(f"asset {asset.asset_id} belongs to another source")
+        if self.source.content_type == "image_note":
+            for block in self.blocks:
+                if block.locator.kind in {"time_range", "video_frame"}:
+                    raise ValueError(
+                        "image_note content must not invent timestamps "
+                        f"(block {block.block_id} has a {block.locator.kind} locator)"
+                    )
+        return self
+
+    @property
+    def content_key(self) -> str:
+        return f"{self.source.source_id}@{self.source_revision}"
+
+
+class TopicGroup(BaseModel):
+    """Topic membership expressed as block ids (time range is optional/derived)."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    topic_id: str = Field(min_length=1)
+    title: str = Field(min_length=1)
+    block_ids: list[str] = Field(min_length=1)
+    time_range: TimeRange | None = None
+    metadata: dict[str, object] = Field(default_factory=dict)
+
+
+class KnowledgeAtomV2(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    atom_id: str = Field(min_length=1)
+    atom_type: AtomType
+    content: str = Field(min_length=1)
+    explanation: str = ""
+    source_ids: list[str] = Field(min_length=1)
+    evidence_ids: list[str] = Field(min_length=1)
+    confidence: float = Field(ge=0, le=1)
+    topic_path: list[str] = Field(default_factory=list)
+    metadata: dict[str, object] = Field(default_factory=dict)
+
+
+class EvidenceRefV2(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    evidence_id: str = Field(min_length=1)
+    source_id: str = Field(min_length=1)
+    source_revision: str = Field(min_length=1)
+    block_id: str = Field(min_length=1)
+    quote: str = Field(min_length=1)
+    locator: Locator
+    processor: Processor
+
+    @property
+    def source_revision_key(self) -> str:
+        return f"{self.source_id}@{self.source_revision}"
+
+
+class ExtractionBundleV2(BaseModel):
+    """The v2 knowledge product: sources + contents + evidence registry + atoms."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    schema_version: Literal[2] = 2
+    sources: list[ContentRef] = Field(default_factory=list)
+    contents: list[ContentBundle] = Field(default_factory=list)
+    evidence_index: dict[str, EvidenceRefV2] = Field(default_factory=dict)
+    topics: list[TopicGroup] = Field(default_factory=list)
+    knowledge: list[KnowledgeAtomV2] = Field(default_factory=list)
+    metadata: dict[str, object] = Field(default_factory=dict)
+    skill_spec: SkillSpec | None = None
+
+    @model_validator(mode="after")
+    def _validate_references(self) -> ExtractionBundleV2:
+        self.validate_references()
+        return self
+
+    # -- reference integrity -------------------------------------------------
+    def content_keys(self) -> list[str]:
+        return [content.content_key for content in self.contents]
+
+    def get_content(self, source_id: str, source_revision: str) -> ContentBundle:
+        for content in self.contents:
+            if content.source.source_id == source_id and content.source_revision == source_revision:
+                return content
+        raise KeyError(f"content not found for {source_id}@{source_revision}")
+
+    def validate_references(self) -> None:
+        source_ids = [source.source_id for source in self.sources]
+        if len(source_ids) != len(set(source_ids)):
+            raise ValidationError.from_exception_data(
+                "ExtractionBundleV2",
+                [{"type": "value_error", "loc": ("sources",), "input": source_ids,
+                  "ctx": {"error": ValueError("duplicate source id in sources")}}],
+            )
+
+        keys = self.content_keys()
+        if len(keys) != len(set(keys)):
+            raise ValueError("duplicate content revision in contents")
+
+        source_map = {source.source_id: source for source in self.sources}
+        blocks_by_key: dict[str, dict[str, ContentBlock]] = {}
+        for content in self.contents:
+            source = source_map.get(content.source.source_id)
+            if source is None:
+                raise ValueError(f"content {content.content_key} references an unregistered source")
+            if source.content_type != content.source.content_type:
+                raise ValueError(f"content {content.content_key} has a conflicting source content type")
+            blocks_by_key[content.content_key] = {
+                block.block_id: block for block in content.blocks
+            }
+        for key, evidence in self.evidence_index.items():
+            if key != evidence.evidence_id:
+                raise ValueError(
+                    f"evidence_index key {key!r} does not match evidence_id {evidence.evidence_id!r}"
+                )
+            revision_key = evidence.source_revision_key
+            if revision_key not in blocks_by_key:
+                raise ValueError(
+                    f"evidence {evidence.evidence_id} references revision {revision_key} "
+                    "that is not present in contents"
+                )
+            if evidence.block_id not in blocks_by_key[revision_key]:
+                raise ValueError(
+                    f"evidence {evidence.evidence_id} references block {evidence.block_id} "
+                    f"that does not exist in revision {revision_key}"
+                )
+            block = blocks_by_key[revision_key][evidence.block_id]
+            if locator_key(block.locator) != locator_key(evidence.locator):
+                raise ValueError(
+                    f"evidence {evidence.evidence_id} locator does not match block {block.block_id}"
+                )
+            block_text = block.raw_text
+            cleaned = block.cleaned_text
+            if evidence.quote not in block_text and (not cleaned or evidence.quote not in cleaned):
+                raise ValueError(
+                    f"evidence {evidence.evidence_id} quote is not present in block "
+                    f"{evidence.block_id} text (fabricated quote)"
+                )
+
+        known_blocks: set[str] = set()
+        for blocks in blocks_by_key.values():
+            known_blocks.update(blocks)
+
+        for atom in self.knowledge:
+            for source_id in atom.source_ids:
+                if source_id not in source_ids:
+                    raise ValueError(
+                        f"atom {atom.atom_id} references source {source_id!r} that is not in sources"
+                    )
+            for evidence_id in atom.evidence_ids:
+                if evidence_id not in self.evidence_index:
+                    raise ValueError(
+                        f"atom {atom.atom_id} references evidence {evidence_id!r} that is not "
+                        "registered in evidence_index"
+                    )
+            evidence_sources = {
+                self.evidence_index[evidence_id].source_id for evidence_id in atom.evidence_ids
+            }
+            if evidence_sources != set(atom.source_ids):
+                raise ValueError(
+                    f"atom {atom.atom_id} source_ids do not match the sources of its evidence"
+                )
+
+        for topic in self.topics:
+            for block_id in topic.block_ids:
+                if block_id not in known_blocks:
+                    raise ValueError(
+                        f"topic {topic.topic_id} references block {block_id!r} that does not exist"
+                    )
+
+    def evidence_for(self, atom: KnowledgeAtomV2) -> list[EvidenceRefV2]:
+        return [self.evidence_index[evidence_id] for evidence_id in atom.evidence_ids]
+
+    def block_map(self) -> dict[str, ContentBlock]:
+        return {
+            block.block_id: block
+            for content in self.contents
+            for block in content.blocks
+        }
+
+
+def dump_bundle_json(bundle: ExtractionBundleV2) -> dict[str, Any]:
+    """Serialize a bundle, keeping the computed ``source_id`` on every ref.
+
+    ``source_id`` is a computed property and is dropped by ``model_dump``; the
+    contract calls it the primary key, so external consumers must see it.
+    """
+    payload = bundle.model_dump(mode="json")
+    for ref, out in zip(bundle.sources, payload.get("sources", []), strict=True):
+        out["source_id"] = ref.source_id
+    for content, cout in zip(bundle.contents, payload.get("contents", []), strict=True):
+        cout.setdefault("source", {})["source_id"] = content.source.source_id
+    return payload

ytk/models/knowledge.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from ytk.models.common import EvidenceRef, TranscriptSource
+from ytk.models.skill import SkillSpec
+from ytk.models.topic import TopicSegment
+
+
+class KnowledgeAtom(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    id: str
+    atom_type: Literal[
+        "terminal_command",
+        "configuration_snippet",
+        "best_practice",
+        "core_concept",
+        "workflow_step",
+        "code_block",
+        "command_sequence",
+        "segment_summary",
+    ]
+    content: str
+    explanation: str
+    evidence: list[EvidenceRef]
+    timestamps: list[float]
+    confidence: float = Field(ge=0, le=1)
+    topic_path: list[str]
+    source_video: str
+
+
+class ExtractionBundle(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    video_id: str
+    metadata: dict[str, object]
+    transcript_source: TranscriptSource
+    topic_segments: list[TopicSegment]
+    knowledge_atoms: list[KnowledgeAtom]
+    skill_spec: SkillSpec