kd-engine 1.0.0__py3-none-any.whl

ytk/pipeline/runners.py

@@ -0,0 +1,578 @@
+"""Pipeline enrichment runners: ASR and OCR glue for the v2 pipeline.
+
+The pipeline itself stays decoupled from model backends; :func:`build_v2_pipeline`
+injects these callables.  Both runners follow the environment rules verified for
+this machine (see ``docs/implementation/five-platform/models.md``):
+
+* **no silent download at inference** — the ASR model is resolved from an
+  explicit local directory and loaded with ``local_files_only=True``; the OCR
+  engine is resolved from explicit ONNX model paths (``YTK_OCR_MODEL_DIR``).
+  Missing models raise :class:`~ytk.errors.ModelUnavailableError` with the
+  preparation command instead of fetching anything (A16).
+* **already-downloaded audio only** — ASR consumes an existing asset from the
+  task directory and never re-downloads (plan T22.1).
+* **chunk-aware output** — faster-whisper's segment timestamps are absolute, so
+  blocks keep real time ranges; long-audio runs are bounded by a deadline.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import math
+import os
+import shutil
+import tempfile
+import time
+from dataclasses import replace
+from importlib.metadata import PackageNotFoundError, version
+from pathlib import Path
+from typing import Any
+
+from ytk.config import AppSettings
+from ytk.errors import ContentIntegrityError, ModelUnavailableError, UserFacingError
+from ytk.ingestion.providers.asr import (
+    AsrChunk, AsrResult, AsrSegment, asr_decoding_profile,
+    missing_asr_model_files as _missing_asr_files,
+)
+from ytk.models.content import (
+    ContentBlock,
+    ContentBundle,
+    TimeRange,
+    TimeRangeLocator,
+    make_block_id,
+)
+from ytk.pipeline.state import ArtifactStore
+
+LOGGER = logging.getLogger(__name__)
+
+#: Candidate local ASR model locations, in preference order.  A multilingual
+#: model is required for Chinese; ``base.en`` must never be the default (A07).
+ASR_MODEL_CANDIDATES: tuple[str, ...] = (
+    "~/.cache/ytk-models/faster-whisper/faster-whisper-large-v3",
+    "~/.cache/ytk-models/faster-whisper/faster-whisper-large-v3-turbo",
+    "~/.cache/ytk-models/faster-whisper/faster-whisper-small",
+    "~/.cache/ytk-models/faster-whisper/faster-whisper-base",
+    "~/.cache/ytk-models/faster-whisper",
+)
+
+#: Model names that are English-only and therefore unusable for Chinese content.
+_ENGLISH_ONLY_MODELS = frozenset({"base.en", "small.en", "medium.en", "tiny.en"})
+
+DEFAULT_OCR_MODEL_DIR = "~/.cache/ytk-models/rapidocr/models"
+ASR_CHUNK_SECONDS = 180.0
+ASR_OVERLAP_SECONDS = 1.0
+ASR_RESUME_SCHEMA_VERSION = 2
+ASR_DURATION_TOLERANCE_SECONDS = 0.5
+
+
+def _sha256_file(path: Path) -> str:
+    import hashlib
+
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _asr_model_fingerprint(model_dir: str) -> dict[str, str]:
+    """Bind recognition to the bytes of the provisioned model and vocabulary."""
+    root = Path(model_dir)
+    if root.is_file():
+        return {root.name: _sha256_file(root)}
+    names = {"model.bin", "config.json", "tokenizer.json", "vocabulary.json", "vocabulary.txt", "preprocessor_config.json"}
+    return {
+        path.name: _sha256_file(path)
+        for path in sorted(root.iterdir())
+        if path.is_file() and path.name in names
+    }
+
+
+def _expand(path: str | None) -> str | None:
+    if not path:
+        return None
+    return os.path.expanduser(path)
+
+
+def resolve_asr_backend(settings: AppSettings) -> str:
+    if settings.asr_backend != "auto":
+        return settings.asr_backend
+    if settings.asr_model:
+        return "whisper_cpp" if Path(settings.asr_model).expanduser().is_file() else "faster_whisper"
+    if settings.whisper_cpp_binary and settings.whisper_cpp_model:
+        return "whisper_cpp"
+    return "faster_whisper"
+
+
+def resolve_asr_model(
+    settings: AppSettings | None = None, *, language: str | None = None,
+) -> str:
+    """Resolve the selected local model; an explicit missing model never falls back."""
+    settings = settings or AppSettings()
+    backend = resolve_asr_backend(settings)
+    explicit = _expand(settings.asr_model)
+    if backend == "whisper_cpp":
+        selected = explicit or _expand(settings.whisper_cpp_model)
+        if not selected or not Path(selected).is_file() or Path(selected).stat().st_size == 0:
+            raise ModelUnavailableError("whisper.cpp requires a provisioned local model file; set YTK_ASR_MODEL or YTK_WHISPER_CPP_MODEL")
+        return selected
+    if explicit:
+        if not Path(explicit).is_dir():
+            raise ModelUnavailableError("the selected ASR model is not a local directory; provision YTK_ASR_MODEL before inference")
+        missing = _missing_asr_files(Path(explicit))
+        if missing:
+            raise ModelUnavailableError(f"local ASR model is incomplete; provision these files before inference: {', '.join(missing)}")
+        return explicit
+    configured = _expand(getattr(settings, "faster_whisper_model", None))
+    candidates: list[str] = []
+    if configured:
+        candidates.append(configured)
+    candidates.extend(os.path.expanduser(path) for path in ASR_MODEL_CANDIDATES)
+    for candidate in candidates:
+        if candidate and Path(candidate).is_dir():
+            name = Path(candidate).name.lower()
+            if name in _ENGLISH_ONLY_MODELS:
+                continue
+            if _missing_asr_files(Path(candidate)):
+                continue
+            return candidate
+    raise ModelUnavailableError(
+        "no local multilingual ASR model is provisioned. Prepare one explicitly, e.g.: "
+        "install the declared optional extra with `python -m pip install 'kd-engine[asr]==1.0.0'` in the connector runtime, then "
+        "explicitly download Systran/faster-whisper-small into a local model directory "
+        "and set YTK_ASR_BACKEND=faster_whisper and YTK_ASR_MODEL to that absolute directory (see connector README). "
+        "The pipeline never downloads models at inference time."
+    )
+
+
+def make_asr_runner(settings: AppSettings | None = None) -> Any:
+    """Build ``asr_runner(content, output_dir) -> list[ContentBlock]``."""
+    settings = settings or AppSettings()
+
+    def run(content: ContentBundle, output_dir: Path) -> Any:
+        return transcribe_content(content, output_dir, settings=settings)
+
+    return run
+
+
+def _checkpoint_plan(payload: dict[str, Any]) -> tuple[tuple[int, float, float], ...] | None:
+    rows = payload.get("chunk_plan")
+    if rows is None and payload.get("resume_schema_version") == 1:
+        # Schema 1 wrote the entire attempted plan after transcribe returned.
+        # Retain that proven layout only when a failed chunk is recorded; it is
+        # still compared with the freshly probed plan before any work is reused.
+        chunks = payload.get("chunk_segments")
+        if not isinstance(chunks, list) or not all(isinstance(row, dict) for row in chunks):
+            return None
+        if not any(row.get("status") == "failed" for row in chunks):
+            return None
+        rows = [[row.get("index"), row.get("start"), row.get("end")] for row in chunks]
+    if not isinstance(rows, list) or not rows:
+        return None
+    plan = []
+    try:
+        for index, row in enumerate(rows):
+            if not isinstance(row, list) or len(row) != 3 or type(row[0]) is not int or row[0] != index:
+                return None
+            start, end = float(row[1]), float(row[2])
+            if not math.isfinite(start) or not math.isfinite(end):
+                return None
+            if abs(start - round(index * (ASR_CHUNK_SECONDS - ASR_OVERLAP_SECONDS), 3)) > 0.001:
+                return None
+            if end < start or (end == start and not (len(rows) == 1 and end == 0)) or end > start + ASR_CHUNK_SECONDS + 0.001:
+                return None
+            if index < len(rows) - 1 and abs(end - (start + ASR_CHUNK_SECONDS)) > 0.001:
+                return None
+            plan.append((index, start, end))
+    except (TypeError, ValueError):
+        return None
+    return tuple(plan)
+
+
+def _reusable_asr_chunks(payload: dict[str, Any], source_id: str) -> dict[int, tuple[AsrSegment, ...]]:
+    """Read proven chunk membership; never infer it from final transcript times."""
+    if payload.get("resume_schema_version") not in {1, ASR_RESUME_SCHEMA_VERSION} or payload.get("source_id") != source_id:
+        return {}
+    if payload.get("metadata", {}).get("complete") is not False:
+        return {}
+    rows = payload.get("chunk_segments")
+    metadata_rows = payload["metadata"].get("chunks")
+    plan = _checkpoint_plan(payload)
+    if plan is None or not isinstance(rows, list) or not rows or not isinstance(metadata_rows, list) or len(metadata_rows) != len(rows):
+        return {}
+    reusable: dict[int, tuple[AsrSegment, ...]] = {}
+    previous_index = -1
+    try:
+        for position, row in enumerate(rows):
+            if not isinstance(row, dict) or type(row.get("index")) is not int:
+                return {}
+            index = row["index"]
+            if index <= previous_index or index >= len(plan):
+                return {}
+            previous_index = index
+            start, end = float(row["start"]), float(row["end"])
+            if (index, start, end) != plan[index]:
+                return {}
+            status = row["status"]
+            if status not in {"ok", "skipped", "failed"} or not isinstance(row.get("segments"), list):
+                return {}
+            recorded = metadata_rows[position]
+            if not isinstance(recorded, dict) or any(recorded.get(key) != row[key] for key in ("index", "start", "end")):
+                return {}
+            if recorded.get("status") != ("ok" if status == "skipped" else status):
+                return {}
+            segments: list[AsrSegment] = []
+            for segment in row["segments"]:
+                text = segment["text"]
+                segment_start, segment_end = float(segment["start"]), float(segment["end"])
+                if not isinstance(text, str) or not text.strip():
+                    return {}
+                if not math.isfinite(segment_start) or not math.isfinite(segment_end):
+                    return {}
+                if segment_start < start - 0.001 or segment_end < segment_start or segment_end > end + 0.001:
+                    return {}
+                segments.append(AsrSegment(text=text, start=segment_start, end=segment_end))
+            if status == "failed":
+                if segments:
+                    return {}
+            else:
+                reusable[index] = tuple(segments)
+    except (KeyError, TypeError, ValueError):
+        return {}
+    return reusable
+
+
+def _asr_result_payload(source_id: str, result: Any, backend: str, language: str | None,
+                        source_duration: float | None = None) -> tuple[list[ContentBlock], dict[str, object], dict[str, Any]]:
+    blocks: list[ContentBlock] = []
+    for order, segment in enumerate(result.segments):
+        text = (segment.text or "").strip()
+        start, end = float(segment.start), float(segment.end)
+        if end < start or not text:
+            continue
+        locator = TimeRangeLocator(time=TimeRange(start=round(start, 3), end=round(end, 3)))
+        blocks.append(ContentBlock(
+            block_id=make_block_id(source_id, locator, text), source_id=source_id,
+            kind="transcript", raw_text=text, language=language, order=order, locator=locator,
+        ))
+    chunks = getattr(result, "chunks", ()) or ()
+    rows = [{"index": c.index, "status": "ok" if c.status == "skipped" else c.status,
+             "start": c.start, "end": c.end, "reused": c.status == "skipped"} for c in chunks]
+    metadata = {
+        "backend": backend, "complete": bool(getattr(result, "complete", True)),
+        "last_chunk_ok": bool(getattr(result, "last_chunk_ok", not (rows and rows[-1]["status"] == "failed"))),
+        "segments": len(blocks), "chunks": rows, "language": language,
+        "duration_seconds": float(getattr(result, "duration", 0.0) or 0.0),
+        "model": getattr(result, "model", None),
+        "audio_sha256": getattr(result, "audio_sha256", None),
+        "decoding_options": dict(getattr(result, "decoding_options", {}) or {}),
+        "segment_diagnostics": [dict(row) for chunk in chunks for row in getattr(chunk, "segment_diagnostics", ())],
+        "diagnostics_interpretation": "decoder_scores_are_not_accuracy_or_fact_verification",
+    }
+    if source_duration is not None:
+        metadata["source_audio_duration_seconds"] = source_duration
+        metadata["audio_duration_matches_source"] = _asr_duration_matches(source_duration, getattr(result, "duration", None))
+    payload = {
+        "resume_schema_version": ASR_RESUME_SCHEMA_VERSION, "source_id": source_id,
+        "blocks": [block.model_dump(mode="json") for block in blocks], "metadata": metadata,
+        "chunk_plan": list(getattr(result, "chunk_plan", ()) or ()),
+        "chunk_segments": [{"index": c.index, "start": c.start, "end": c.end, "status": c.status,
+                            "segments": [{"text": s.text, "start": s.start, "end": s.end} for s in c.segments]}
+                           for c in chunks],
+    }
+    return blocks, metadata, payload
+
+
+def _asr_duration_matches(expected: float, actual: object) -> bool:
+    return (isinstance(actual, (int, float)) and not isinstance(actual, bool)
+            and math.isfinite(actual) and actual > 0
+            and abs(expected - actual) <= ASR_DURATION_TOLERANCE_SECONDS)
+
+
+def _verified_asr_audio(media: Any, source_path: Path, audio_path: Path) -> float:
+    """Publish only a validated complete conversion; an existing WAV is untrusted."""
+    source_info = media.probe(source_path)
+    duration = getattr(source_info, "audio_duration_seconds", None)
+    if duration is None:
+        duration = source_info.duration_seconds
+    if not _asr_duration_matches(duration or 0, duration) or not source_info.has_audio:
+        raise ContentIntegrityError("Cannot verify the source audio duration before transcription")
+
+    def valid_audio(path: Path) -> bool:
+        if not path.is_file() or path.stat().st_size == 0:
+            return False
+        try:
+            info = media.probe(path)
+            return (info.has_audio and info.sample_rate == 16000 and info.channels == 1
+                    and _asr_duration_matches(duration, info.duration_seconds))
+        except Exception:
+            return False
+
+    if valid_audio(audio_path):
+        return float(duration)
+    audio_path.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.NamedTemporaryFile(prefix=".asr-conversion-", suffix=".wav", dir=audio_path.parent, delete=False) as handle:
+        preparing = Path(handle.name)
+    try:
+        media.to_asr_audio(source_path, preparing)
+        if not valid_audio(preparing):
+            raise ContentIntegrityError("Converted ASR audio duration does not match the source; transcription was stopped")
+        os.replace(preparing, audio_path)
+    finally:
+        preparing.unlink(missing_ok=True)
+    return float(duration)
+
+
+def transcribe_content(
+    content: ContentBundle,
+    output_dir: Path,
+    *,
+    settings: AppSettings | None = None,
+    model: Any | None = None,
+    language: str | None = None,
+) -> tuple[list[ContentBlock], dict[str, object]]:
+    """Transcribe one already-downloaded audio/video asset into timed blocks.
+
+    Returns ``(blocks, asr_metadata)``; the v2 pipeline already consumes both
+    shapes via ``_split_runner_output`` (the tuple return is the newer contract
+    introduced with the ASR metadata plumbing)."""
+    settings = settings or AppSettings()
+    language = language if language is not None else settings.asr_language
+    language = None if language == "auto" else language
+    decoding_options = asr_decoding_profile(settings, language)
+    source_id = content.source.source_id
+    asset = _pick_audio_asset(content)
+    if asset is None:
+        raise UserFacingError(f"no audio/video asset to transcribe for {source_id}")
+    source_path = Path(output_dir) / asset.relative_path
+    if not source_path.is_file():
+        raise UserFacingError(f"audio asset missing on disk: {source_path}")
+    backend = resolve_asr_backend(settings)
+    if backend == "whisper_cpp":
+        if settings.asr_beam_size != 5 or settings.asr_initial_prompt or settings.asr_hotwords:
+            raise UserFacingError("ASR beam size and vocabulary hints require the faster_whisper backend")
+        decoding_options = {"language": language or "auto", "word_timestamps": False}
+    model_dir = resolve_asr_model(settings, language=language)
+
+    from ytk.ingestion.media import MediaService
+
+    media = MediaService(settings=settings)
+    audio_path = source_path
+    source_duration: float | None = None
+    if asset.media_type == "video" or backend == "whisper_cpp":
+        audio_path = Path(output_dir) / "artifacts" / f"{source_id.replace(':', '-')}-{_sha256_file(source_path)[:20]}-asr.wav"
+        source_duration = _verified_asr_audio(media, source_path, audio_path)
+
+    runtime: str | None = None
+    if backend == "whisper_cpp":
+        binary = settings.whisper_cpp_binary
+        runtime = shutil.which(binary) if binary else None
+        if runtime is None:
+            raise ModelUnavailableError("whisper.cpp executable is unavailable; configure YTK_WHISPER_CPP_BINARY before inference")
+        runtime_fingerprint = _sha256_file(Path(runtime))
+    else:
+        try:
+            runtime_fingerprint = version("faster-whisper")
+        except PackageNotFoundError:
+            runtime_fingerprint = "not-installed"
+
+    # Layered cache (A19/A33): recognition is keyed by audio digest + model dir +
+    # language; a model or language change invalidates, a signed-URL change does not.
+    cache_parts: dict[str, object] | None = None
+    cache: Any | None = None
+    reuse_chunks: dict[int, tuple[AsrSegment, ...]] = {}
+    resume_plan: tuple[tuple[int, float, float], ...] | None = None
+    restored_language: str | None = None
+    restored_diagnostics: dict[int, tuple[dict[str, Any], ...]] = {}
+    try:
+        from ytk.cache.v2 import LayeredCache
+
+        cache = LayeredCache(root=settings.cache_dir, enabled=True)
+        cache_parts = {
+            "source_id": source_id,
+            "audio_sha256": _sha256_file(audio_path),
+            "model": model_dir,
+            "backend": backend,
+            "runtime": runtime_fingerprint,
+            "model_files": _asr_model_fingerprint(model_dir),
+            "processor_version": "ytk-local-asr-v3",
+            "language": language or "auto",
+            "decoding_options": decoding_options,
+            "chunk_seconds": ASR_CHUNK_SECONDS,
+            "overlap_seconds": ASR_OVERLAP_SECONDS,
+        }
+        cached = cache.get("asr", cache_parts)
+        if cached:
+            payload = json.loads(cached.decode("utf-8"))
+            cached_blocks = [ContentBlock(**row) for row in payload["blocks"]]
+            if (
+                all(block.source_id == source_id for block in cached_blocks)
+                and payload["metadata"].get("complete") is True
+                and (source_duration is None or _asr_duration_matches(source_duration, payload["metadata"].get("duration_seconds")))
+            ):
+                LOGGER.info("ASR cache hit for %s (%d blocks)", source_id, len(cached_blocks))
+                return cached_blocks, payload["metadata"]
+            if all(block.source_id == source_id for block in cached_blocks):
+                reuse_chunks = _reusable_asr_chunks(payload, source_id)
+                resume_plan = _checkpoint_plan(payload) if reuse_chunks else None
+                prior_language = payload["metadata"].get("language")
+                if reuse_chunks and isinstance(prior_language, str):
+                    restored_language = prior_language
+                for index in reuse_chunks:
+                    restored_diagnostics[index] = tuple(
+                        row for row in payload["metadata"].get("segment_diagnostics", [])
+                        if isinstance(row, dict) and row.get("chunk_index") == index
+                    )
+    except Exception as cache_exc:  # cache must never break recognition
+        LOGGER.debug("ASR cache unavailable: %s", cache_exc)
+        cache = None
+        cache_parts = None
+
+    from ytk.ingestion.providers.asr import LocalAsrService, WhisperCppAsrProvider
+
+    transcriber = model if callable(model) else None
+    if backend == "whisper_cpp" and transcriber is None:
+        started = time.monotonic()
+
+        def transcribe_cpp(path: Path, selected_language: str | None) -> dict[str, object]:
+            remaining = settings.task_deadline_s - (time.monotonic() - started)
+            if remaining <= 0:
+                raise TimeoutError("whisper.cpp ASR deadline exceeded")
+            provider = WhisperCppAsrProvider(
+                metadata_fetcher=lambda _: {}, whisper_cpp_binary=runtime,
+                model_path=model_dir, language=selected_language or "auto",
+                output_dir=path.parent,
+            )
+            payload = provider._run_whisper_cpp(path, timeout=remaining)
+            segments = provider._parse_segments(payload)
+            return {
+                "language": selected_language,
+                "segments": [
+                    {"text": segment.text, "start": segment.start, "end": segment.end}
+                    for segment in segments
+                ],
+            }
+
+        transcriber = transcribe_cpp
+
+    def persist_progress(result: AsrResult) -> None:
+        if cache is None or cache_parts is None:
+            return
+        try:
+            # Keep later successful chunks while retrying an earlier failure;
+            # another interruption must not discard their original membership.
+            if resume_plan == result.chunk_plan:
+                saved = {index: AsrChunk(index=index, start=resume_plan[index][1], end=resume_plan[index][2],
+                                        status="skipped", attempts=0, segments=segments)
+                         for index, segments in reuse_chunks.items()}
+                saved.update({chunk.index: chunk for chunk in result.chunks})
+                result = replace(result, chunks=tuple(saved[index] for index in sorted(saved)))
+            result = with_diagnostics(result)
+            detected = result.language or restored_language or language
+            _, _, payload = _asr_result_payload(source_id, result, backend, detected, source_duration)
+            cache.put("asr", cache_parts, json.dumps(payload, ensure_ascii=False).encode("utf-8"))
+        except Exception as cache_exc:  # cache must never break recognition
+            LOGGER.debug("ASR checkpoint store failed: %s", cache_exc)
+
+    def with_diagnostics(result: AsrResult) -> AsrResult:
+        return replace(result, decoding_options=decoding_options, chunks=tuple(
+            replace(chunk, segment_diagnostics=restored_diagnostics.get(chunk.index, ()))
+            if chunk.status == "skipped" and not chunk.segment_diagnostics else chunk
+            for chunk in result.chunks
+        ))
+
+    service = LocalAsrService(
+        model_path=model_dir if backend == "faster_whisper" else None,
+        settings=settings,
+        media=media,
+        transcriber=transcriber,
+        language=language,
+        model_candidates=(),
+        chunk_seconds=ASR_CHUNK_SECONDS,
+        overlap_seconds=ASR_OVERLAP_SECONDS,
+        on_progress=persist_progress,
+        resume_plan=resume_plan,
+    )
+    result = (
+        service.transcribe(audio_path, reuse_chunks=reuse_chunks)
+        if reuse_chunks else service.transcribe(audio_path)
+    )
+    result = with_diagnostics(result)
+    if source_duration is not None and not _asr_duration_matches(source_duration, getattr(result, "duration", None)):
+        raise ContentIntegrityError("Recognized audio duration does not match the source; a complete transcript cannot be confirmed")
+    detected_language = getattr(result, "language", None) or restored_language or language
+    blocks, asr_metadata, payload = _asr_result_payload(source_id, result, backend, detected_language, source_duration)
+    if cache is not None and cache_parts is not None:
+        try:
+            cache.put(
+                "asr",
+                cache_parts,
+                json.dumps(payload, ensure_ascii=False).encode("utf-8"),
+            )
+        except Exception as cache_exc:  # cache must never break recognition
+            LOGGER.debug("ASR cache store failed: %s", cache_exc)
+    return blocks, asr_metadata
+
+
+def _pick_audio_asset(content: ContentBundle) -> Any | None:
+    audio = [asset for asset in content.assets if asset.media_type == "audio"]
+    if audio:
+        return sorted(audio, key=lambda asset: asset.order)[0]
+    video = [asset for asset in content.assets if asset.media_type == "video"]
+    if video:
+        return sorted(video, key=lambda asset: asset.order)[0]
+    return None
+
+
+def make_ocr_runner(settings: AppSettings | None = None) -> Any:
+    """Build ``ocr_runner(content, output_dir) -> list[ContentBlock]``.
+
+    Raises :class:`ModelUnavailableError` when the engine/models are absent so
+    the pipeline marks the run PARTIAL/blocked with an actionable reason rather
+    than silently producing an OCR-less image-note doc (A16/A09).
+    """
+    settings = settings or AppSettings()
+
+    def run(content: ContentBundle, output_dir: Path) -> Any:
+        image_assets = [asset for asset in content.assets if asset.media_type == "image"]
+        if not image_assets:
+            return []
+        model_dir = _expand(getattr(settings, "ocr_model_dir", None)) or os.path.expanduser(
+            DEFAULT_OCR_MODEL_DIR
+        )
+        from ytk.cache.v2 import LayeredCache
+        from ytk.ocr.engine import get_engine
+        from ytk.ocr.pipeline import run_ocr_for_content
+
+        diagnostics: list[str] = []
+        blocks = run_ocr_for_content(
+            content, Path(output_dir), engine=get_engine(model_dir=model_dir), diagnostics=diagnostics,
+            cache=LayeredCache(root=settings.cache_dir, enabled=True),
+        )
+        for diagnostic in diagnostics:
+            LOGGER.warning("OCR: %s", diagnostic)
+        return blocks, {"complete": not diagnostics, "diagnostics": diagnostics}
+
+    return run
+
+
+def make_artifact_store_factory() -> Any:
+    """Factory used by the pipeline to get a locked store per task directory."""
+
+    def factory(root: Path) -> ArtifactStore:
+        return ArtifactStore(Path(root))
+
+    return factory
+
+
+__all__ = [
+    "ASR_MODEL_CANDIDATES",
+    "DEFAULT_OCR_MODEL_DIR",
+    "make_artifact_store_factory",
+    "make_asr_runner",
+    "make_ocr_runner",
+    "resolve_asr_model",
+    "resolve_asr_backend",
+    "transcribe_content",
+]