kd-engine 1.0.0__py3-none-any.whl

ytk/batch/runner.py

@@ -0,0 +1,1149 @@
+"""v2 batch execution (T40.1/T40.2).
+
+Identity, directories and resume are driven by the canonical source id, never
+by input order:
+
+* output dir = ``<out>/<platform>/<stable-source-id>`` (readable title may only
+  ever be a suffix, never part of the key);
+* legacy ``video-NNN`` dirs are *discovered* (strict identity verification,
+  never bulk-renamed) so an interrupted pre-v2 batch can still be resumed;
+* duplicates are removed after canonicalization (BV/av/short-long chains that
+  resolve to the same source are processed once) and the original alias is
+  recorded in ``.batch-aliases.json``;
+* resume decisions are per format: verified manifest outputs (``output_valid``),
+  justified ``not_applicable``/``NO_KNOWLEDGE`` judgments, and legacy marker
+  coverage.  A corrupt artifact can never be skipped; ``--refresh`` re-runs the
+  requested formats but never hides an unresolved remote submission;
+* one blocked platform/source never aborts the remaining sources; the report
+  counts per platform / format / status and the exit code is 1 iff any source
+  failed or was blocked.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import inspect
+import json
+import logging
+import re
+import time
+from collections import Counter
+from collections.abc import Callable, Mapping, MutableMapping, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from ytk.config import AppSettings
+from ytk.errors import UserFacingError
+from ytk.ingestion.resolver import resolve_source
+from ytk.logging import redact_text, redact_url
+from ytk.models.source import SUPPORTED_PLATFORMS, ContentRef
+from ytk.pipeline.state import (
+    MANIFEST_FILENAME,
+    OUTPUT_VALIDATOR_VERSION,
+    OVERALL_BLOCKED,
+    OVERALL_COMPLETE,
+    OVERALL_FAILED,
+    OVERALL_NO_KNOWLEDGE,
+    OVERALL_PARTIAL,
+    REASON_OUTPUTS_NOT_APPLICABLE,
+    ArtifactStore,
+    ManifestV2,
+    derive_overall_status,
+    manifest_recorded_outcome,
+    output_entry_claims_no_knowledge,
+)
+
+LOGGER = logging.getLogger(__name__)
+
+BATCH_MARKER_FILENAME = ".batch-complete.json"
+BATCH_ALIASES_FILENAME = ".batch-aliases.json"
+BATCH_REPORT_FILENAME = ".batch-report.json"
+BATCH_MARKER_SCHEMA = 2
+
+#: Validator version of the v2 pipeline outputs (kept in sync with pipeline/v2.py).
+V2_VALIDATOR_VERSION = OUTPUT_VALIDATOR_VERSION
+
+#: Resume diagnostics recorded when a derived completion cannot be trusted.
+RESUME_REASON_STALE_MARKER = "STALE_MARKER"
+RESUME_REASON_MANIFEST_MISSING = "MANIFEST_MISSING"
+
+_STATUS_COMPLETE = "complete"
+_STATUS_PARTIAL = "partial"
+_STATUS_NO_KNOWLEDGE = "no_knowledge"
+_STATUS_NOT_APPLICABLE = "not_applicable"
+_STATUS_SKIPPED = "skipped"
+_STATUS_FAILED = "failed"
+_STATUS_BLOCKED = "blocked"
+
+_FAILED_STATUSES = frozenset({_STATUS_FAILED, _STATUS_BLOCKED})
+
+_OUTCOME_TO_STATUS: dict[str, str] = {
+    OVERALL_COMPLETE: _STATUS_COMPLETE,
+    OVERALL_PARTIAL: _STATUS_PARTIAL,
+    OVERALL_NO_KNOWLEDGE: _STATUS_NO_KNOWLEDGE,
+    OVERALL_BLOCKED: _STATUS_BLOCKED,
+    OVERALL_FAILED: _STATUS_FAILED,
+    REASON_OUTPUTS_NOT_APPLICABLE: _STATUS_FAILED,
+}
+
+#: Categories that mean "cannot proceed here" rather than "our bug".
+_BLOCKED_CATEGORIES = frozenset(
+    {
+        "auth_required",
+        "permission_denied",
+        "content_removed",
+        "rate_limited",
+        "model_missing",
+        "unknown_task",
+        "service_unavailable",
+    }
+)
+
+
+# --------------------------------------------------------------------------- #
+# Result records
+# --------------------------------------------------------------------------- #
+@dataclass
+class SourceResult:
+    """One input value's batch outcome (redacted before it reaches the report)."""
+
+    index: int
+    input_value: str
+    source_id: str | None = None
+    identity_key: str | None = None
+    acquired_source_id: str | None = None
+    platform: str | None = None
+    output_dir: str | None = None
+    status: str = _STATUS_FAILED
+    outcome: str | None = None
+    formats: dict[str, str] = field(default_factory=dict)
+    requested_formats: list[str] = field(default_factory=list)
+    error_code: str | None = None
+    error: str | None = None
+    resume_reason: str | None = None
+    resumed: bool = False
+    duration_ms: int = 0
+
+    @property
+    def failed(self) -> bool:
+        return self.status in _FAILED_STATUSES
+
+    @property
+    def skipped(self) -> bool:
+        return self.status == _STATUS_SKIPPED
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "index": self.index,
+            "input": self.input_value,
+            "source_id": self.source_id,
+            "identity_key": self.identity_key,
+            "acquired_source_id": self.acquired_source_id,
+            "platform": self.platform,
+            "output_dir": self.output_dir,
+            "status": self.status,
+            "outcome": self.outcome,
+            "formats": dict(self.formats),
+            "requested_formats": list(self.requested_formats),
+            "error_code": self.error_code,
+            "error": self.error,
+            "resume_reason": self.resume_reason,
+            "resumed": self.resumed,
+            "duration_ms": self.duration_ms,
+        }
+
+
+@dataclass
+class BatchReport:
+    """Per-source results plus the counters the CLI/CI print."""
+
+    out_root: Path
+    results: list[SourceResult] = field(default_factory=list)
+    started_at: float = field(default_factory=time.time)
+    finished_at: float = 0.0
+
+    @property
+    def total(self) -> int:
+        return len(self.results)
+
+    @property
+    def processed(self) -> int:
+        """Sources that actually executed (resume skips are not processed)."""
+        return sum(
+            1
+            for result in self.results
+            if not result.failed and not result.skipped and not result.resumed
+        )
+
+    @property
+    def skipped(self) -> int:
+        """Duplicates plus resume skips (historical summary semantics)."""
+        return sum(1 for result in self.results if result.skipped or result.resumed)
+
+    @property
+    def failed(self) -> int:
+        return sum(1 for result in self.results if result.failed)
+
+    @property
+    def exit_code(self) -> int:
+        return 1 if self.failed else 0
+
+    def status_counts(self) -> dict[str, int]:
+        return dict(Counter(result.status for result in self.results))
+
+    def platform_counts(self) -> dict[str, dict[str, int]]:
+        counts: dict[str, Counter[str]] = {}
+        for result in self.results:
+            platform = result.platform or "unknown"
+            counts.setdefault(platform, Counter())[result.status] += 1
+        return {platform: dict(counter) for platform, counter in sorted(counts.items())}
+
+    def format_counts(self) -> dict[str, dict[str, int]]:
+        counts: dict[str, Counter[str]] = {}
+        for result in self.results:
+            for fmt in result.requested_formats:
+                status = result.formats.get(fmt, "not_run")
+                counts.setdefault(fmt, Counter())[status] += 1
+        return {fmt: dict(counter) for fmt, counter in sorted(counts.items())}
+
+    def legacy_summary_line(self) -> str:
+        """The historical one-line summary (kept for scripts that parse it)."""
+        return (
+            f"Batch Complete: {self.processed} processed, {self.skipped} skipped, "
+            f"{self.failed} failed (of {self.total} total)"
+        )
+
+    def summary_lines(self) -> list[str]:
+        lines = [self.legacy_summary_line()]
+        status_counts = self.status_counts()
+        lines.append(
+            "Status counts: "
+            + ", ".join(f"{status}={count}" for status, count in sorted(status_counts.items()))
+        )
+        for platform, counts in self.platform_counts().items():
+            detail = ", ".join(f"{status}={count}" for status, count in sorted(counts.items()))
+            lines.append(f"  platform {platform}: {detail}")
+        for fmt, counts in self.format_counts().items():
+            detail = ", ".join(f"{status}={count}" for status, count in sorted(counts.items()))
+            lines.append(f"  format {fmt}: {detail}")
+        for result in self.results:
+            label = result.source_id or result.input_value
+            suffix = f" -> {result.output_dir}" if result.output_dir else ""
+            if result.failed:
+                lines.append(
+                    f"  [{result.index}/{self.total}] {result.status.upper()}: "
+                    f"{result.input_value} ({label}){suffix}"
+                    f"{' — ' + result.error if result.error else ''}"
+                )
+            elif result.resumed or result.skipped:
+                lines.append(f"  [{result.index}/{self.total}] {result.status}: {label}{suffix}")
+            if result.resume_reason:
+                lines.append(
+                    f"  [{result.index}/{self.total}] resume invalidated "
+                    f"({result.resume_reason}): {label}{suffix}"
+                )
+        return lines
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema": 1,
+            "out_root": str(self.out_root),
+            "total": self.total,
+            "processed": self.processed,
+            "skipped": self.skipped,
+            "failed": self.failed,
+            "exit_code": self.exit_code,
+            "status_counts": self.status_counts(),
+            "platform_counts": self.platform_counts(),
+            "format_counts": self.format_counts(),
+            "started_at": self.started_at,
+            "finished_at": self.finished_at,
+            "results": [result.to_dict() for result in self.results],
+        }
+
+
+# --------------------------------------------------------------------------- #
+# Identity / directory helpers
+# --------------------------------------------------------------------------- #
+def resolve_batch_identity(
+    value: str, *, redirect_resolver: Callable[[str], str] | None = None
+) -> ContentRef:
+    """Canonical identity for one batch input (raises on unresolvable links)."""
+    return resolve_source(value, redirect_resolver=redirect_resolver)
+
+
+def _normalise_display_part(value: Any) -> str | None:
+    """Canonical, filesystem-safe display part (Bilibili ``p``), or ``None``."""
+    if value in (None, ""):
+        return None
+    text = str(value).strip()
+    if not text:
+        return None
+    if text.isdigit():
+        number = int(text)
+        return str(number) if number >= 0 else None
+    safe = re.sub(r"[^A-Za-z0-9._-]+", "-", text).strip("-._")
+    return safe or None
+
+
+def display_part(ref: ContentRef) -> str | None:
+    """The part selector carried by the ref (Bilibili ``?p=N``), if any."""
+    display = ref.display or {}
+    return _normalise_display_part(display.get("p"))
+
+
+def batch_identity_key(ref: ContentRef) -> str:
+    """Dedupe/identity key for a batch input.
+
+    Distinct Bilibili parts (``?p=1`` vs ``?p=2``) resolve to the same
+    ``source_id`` *before acquisition* (the ``cid`` variant only appears after
+    the API call), so the display part is part of the batch identity.  Without
+    this, the second part would be dropped as a duplicate (R-08/A03).
+    """
+    part = display_part(ref)
+    if part is not None:
+        return f"{ref.source_id}#p={part}"
+    return ref.source_id
+
+
+def stable_source_dirname(ref: ContentRef) -> str:
+    """Filesystem-safe, stable directory name for one source (title-free)."""
+    raw = ref.content_id
+    part = display_part(ref)
+    if part is not None:
+        raw = f"{raw}-p{part}"
+    if ref.variant_id:
+        raw = f"{raw}--{ref.variant_id}"
+    safe = re.sub(r"[^A-Za-z0-9._-]+", "-", raw).strip("-._") or "source"
+    if ".." in safe:  # never expose a traversal-looking component
+        safe = safe.replace("..", "-").strip("-._") or "source"
+    if safe != raw or len(safe) > 100:
+        digest = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:8]
+        safe = f"{safe[:100]}-{digest}"
+    return safe
+
+
+def stable_output_dir(out_root: Path | str, ref: ContentRef) -> Path:
+    """``<out>/<platform>/<stable-id>`` — the only key for a new batch."""
+    return Path(out_root) / ref.platform / stable_source_dirname(ref)
+
+
+def default_formats_for(platform: str) -> set[str]:
+    """Per-platform default formats (YouTube keeps its legacy triad)."""
+    from ytk.ingestion.registry import PlatformRegistry
+
+    try:
+        registration = PlatformRegistry.default().get(platform)
+    except UserFacingError:
+        return {"json", "md"}
+    return set(registration.default_formats)
+
+
+def requested_formats_for(
+    ref: ContentRef, *, formats: set[str] | None, formats_explicit: bool
+) -> set[str]:
+    if formats_explicit and formats:
+        return set(formats)
+    return default_formats_for(ref.platform)
+
+
+# --------------------------------------------------------------------------- #
+# Marker helpers (the marker is a *derived* cache; the manifest is authority)
+# --------------------------------------------------------------------------- #
+def read_batch_marker(directory: Path | str) -> dict[str, Any] | None:
+    path = Path(directory) / BATCH_MARKER_FILENAME
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except FileNotFoundError:
+        return None
+    except (OSError, json.JSONDecodeError):
+        return None
+    if not isinstance(payload, dict):
+        return None
+    if payload.get("schema") not in (1, BATCH_MARKER_SCHEMA):
+        return None
+    return payload
+
+
+def marker_source_id(
+    marker: Mapping[str, Any] | None,
+    *,
+    redirect_resolver: Callable[[str], str] | None = None,
+) -> str | None:
+    """Strict identity from a marker: a parseable source_id or resolvable URL."""
+    if not marker:
+        return None
+    source_id = marker.get("source_id")
+    if isinstance(source_id, str) and ":" in source_id:
+        platform = source_id.split(":", 1)[0]
+        if platform in SUPPORTED_PLATFORMS:
+            return source_id
+    url = marker.get("url")
+    if isinstance(url, str) and url:
+        try:
+            return resolve_source(url, redirect_resolver=redirect_resolver).source_id
+        except (UserFacingError, ValueError):
+            return None
+    return None
+
+
+def legacy_output_dirs(
+    out_root: Path | str,
+    *,
+    redirect_resolver: Callable[[str], str] | None = None,
+) -> dict[str, Path]:
+    """Map ``source_id -> legacy video-NNN dir`` after strict identity checks.
+
+    Discovery only — legacy directories are never renamed or overwritten.
+    """
+    mapping: dict[str, Path] = {}
+    root = Path(out_root)
+    if not root.is_dir():
+        return mapping
+    for directory in sorted(root.glob("video-*")):
+        if not directory.is_dir():
+            continue
+        marker = read_batch_marker(directory)
+        source_id = marker_source_id(marker, redirect_resolver=redirect_resolver)
+        if source_id is not None:
+            mapping.setdefault(source_id, directory)
+    return mapping
+
+
+def _legacy_marker_covered(
+    marker: Mapping[str, Any] | None,
+    requested: set[str],
+    directory: Path,
+    *,
+    doc_verifier: Callable[[Path], bool] | None,
+) -> set[str]:
+    """Formats an old marker covers — with the historical doc-verification rule."""
+    if not marker:
+        return set()
+    covered = {str(fmt) for fmt in (marker.get("formats") or []) if isinstance(fmt, str)}
+    covered &= requested
+    if "doc" in covered and doc_verifier is not None and not doc_verifier(directory):
+        covered.discard("doc")
+    return covered
+
+
+def write_batch_marker(
+    directory: Path | str,
+    *,
+    ref: ContentRef,
+    requested: Sequence[str],
+    format_status: Mapping[str, str],
+    outcome: str,
+    manifest: ManifestV2 | None = None,
+) -> Path:
+    """Write the derived resume marker (never the primary completion authority)."""
+    payload: dict[str, Any] = {
+        "schema": BATCH_MARKER_SCHEMA,
+        "source_id": ref.source_id,
+        "platform": ref.platform,
+        "content_type": ref.content_type,
+        "url": redact_url(ref.canonical_url),
+        "formats": sorted(requested),
+        "format_status": dict(format_status),
+        "outcome": outcome,
+        "validator_version": V2_VALIDATOR_VERSION,
+        "verified": manifest is not None,
+        "ts": time.time(),
+    }
+    if manifest is not None:
+        payload["request_fingerprint"] = manifest.request_fingerprint
+        payload["source_revision"] = manifest.source_revision
+        payload["refresh_epoch"] = manifest.refresh_epoch
+        payload["manifest"] = MANIFEST_FILENAME
+    store = ArtifactStore(directory)
+    return store.write_json(BATCH_MARKER_FILENAME, payload)
+
+
+def _manifest_for(directory: Path) -> ManifestV2 | None:
+    try:
+        return ArtifactStore(directory).read_manifest()
+    except Exception:  # noqa: BLE001 - a bad manifest is simply "no manifest"
+        return None
+
+
+def _status_from_manifest(manifest: ManifestV2, requested: Sequence[str]) -> tuple[str, str]:
+    """``(batch status, outcome)`` derived from a manifest (authority)."""
+    overall, _reasons = derive_overall_status(manifest, requested=requested)
+    status = _OUTCOME_TO_STATUS.get(overall, _STATUS_FAILED)
+    if overall == OVERALL_FAILED and _all_not_applicable(manifest, requested):
+        return status, REASON_OUTPUTS_NOT_APPLICABLE
+    return status, overall
+
+
+def _manifest_terminal_judgment(
+    manifest: ManifestV2, requested: Sequence[str]
+) -> tuple[str, str] | None:
+    """Reuse a deterministic all-not-applicable verdict recorded by the pipeline.
+
+    The pipeline writes ``format_status=not_applicable`` for every requested
+    format plus a top-level ``outcome`` (extra key) when the content is
+    NO_KNOWLEDGE or every requested format is inapplicable.  Both verdicts are
+    content-level and stable for the same source revision + validator version,
+    so resuming must not re-dispatch them (A30).  Missing/older metadata fails
+    closed (``None`` → the format is re-run).
+    """
+    entries = [manifest.outputs.get(fmt) for fmt in requested]
+    if not entries or any(entry is None for entry in entries):
+        return None
+    assert all(entry is not None for entry in entries)
+    if not all(entry.status == "not_applicable" for entry in entries if entry is not None):
+        return None
+    if not all(
+        entry.validator_version == V2_VALIDATOR_VERSION
+        for entry in entries
+        if entry is not None
+    ):
+        return None
+    if not _judgment_evidence_valid(manifest, entries):
+        return None
+    recorded_outcome = manifest_recorded_outcome(manifest)
+    claims_no_knowledge = recorded_outcome == OVERALL_NO_KNOWLEDGE or all(
+        entry is not None and output_entry_claims_no_knowledge(entry)
+        for entry in entries
+    )
+    if claims_no_knowledge:
+        return (_STATUS_NO_KNOWLEDGE, OVERALL_NO_KNOWLEDGE)
+    if recorded_outcome == OVERALL_FAILED:
+        return (_STATUS_FAILED, REASON_OUTPUTS_NOT_APPLICABLE)
+    return None
+
+
+def _judgment_evidence_valid(
+    manifest: ManifestV2, entries: Sequence[Any]
+) -> bool:
+    for entry in entries:
+        if entry is None:
+            return False
+        evidence = getattr(entry, "evidence_fingerprint", None)
+        if evidence:
+            if not manifest.source_revision or evidence != manifest.source_revision:
+                return False
+    return True
+
+
+def _all_not_applicable(manifest: ManifestV2, requested: Sequence[str]) -> bool:
+    entries = [manifest.outputs.get(fmt) for fmt in requested]
+    return bool(entries) and all(
+        entry is not None and entry.status == "not_applicable" for entry in entries
+    )
+
+
+def _manifest_covered_formats(
+    store: ArtifactStore,
+    manifest: ManifestV2,
+    requested: Sequence[str],
+) -> set[str]:
+    """Formats already terminal under the current manifest (fail-closed)."""
+    covered: set[str] = set()
+    for fmt in requested:
+        skip, _reason = store.should_skip_format(
+            fmt,
+            request_fingerprint=manifest.request_fingerprint,
+            evidence_fingerprint=manifest.source_revision,
+            checker_version=(
+                manifest.outputs[fmt].checker_version
+                if fmt in manifest.outputs
+                else None
+            ),
+            validator_version=V2_VALIDATOR_VERSION,
+            source_revision=manifest.source_revision,
+        )
+        if skip:
+            covered.add(fmt)
+    return covered
+
+
+# --------------------------------------------------------------------------- #
+# Pipeline invocation
+# --------------------------------------------------------------------------- #
+def _call_pipeline(
+    pipeline: Any,
+    *,
+    value: str,
+    output_dir: Path,
+    formats: set[str] | None,
+    fixture_transcript: Path | None,
+    fixture_content: Path | None,
+    doc_language: str | None,
+    with_reviewer_scaffolding: bool,
+    refresh: bool,
+    ref: ContentRef | None,
+    preserve_existing_formats: bool = False,
+) -> Any:
+    """Call either the v1 or the v2 pipeline using only supported kwargs."""
+    parameters: Any
+    try:
+        parameters = inspect.signature(pipeline.run).parameters
+    except (TypeError, ValueError):  # pragma: no cover - exotic callables
+        parameters = {}
+    accepts_extra = any(
+        parameter.kind is inspect.Parameter.VAR_KEYWORD
+        for parameter in parameters.values()
+    )
+    kwargs: dict[str, Any] = {"url": value, "output_dir": output_dir}
+    candidates: dict[str, Any] = {
+        "formats": formats,
+        "fixture_transcript": fixture_transcript,
+        "fixture_content": fixture_content,
+        "doc_language": doc_language,
+        "with_reviewer_scaffolding": with_reviewer_scaffolding,
+        "refresh": refresh,
+        "formats_explicit": formats is not None,
+        "platform": ref.platform if ref is not None else None,
+        "content_type": ref.content_type if ref is not None else None,
+        "preserve_existing_formats": preserve_existing_formats,
+    }
+    for name, candidate in candidates.items():
+        if candidate is not None and (name in parameters or accepts_extra):
+            kwargs[name] = candidate
+    return pipeline.run(**kwargs)
+
+
+def _interpret_v2_result(
+    result: Any, requested: Sequence[str], directory: Path
+) -> tuple[str, str, dict[str, str], list[str]]:
+    """Return ``(status, outcome, format_status, reasons)`` for a v2 run."""
+
+    def _field(name: str) -> Any:
+        if isinstance(result, Mapping):
+            return result.get(name)
+        return getattr(result, name, None)
+
+    outcome_raw = _field("outcome")
+    outcome = str(outcome_raw).upper() if outcome_raw is not None else ""
+    format_status = dict(_field("format_status") or {})
+    reasons = [str(reason) for reason in (_field("reasons") or [])]
+
+    manifest = _manifest_for(directory)
+    if manifest is not None:
+        # Manifest outputs are the authority when present; marker is derived.
+        authoritative = {
+            fmt: manifest.outputs[fmt].status
+            for fmt in requested
+            if fmt in manifest.outputs
+        }
+        if authoritative:
+            format_status = {**format_status, **authoritative}
+        if not outcome:
+            outcome, manifest_reasons = _status_from_manifest(manifest, requested)
+            reasons.extend(manifest_reasons)
+    if not outcome:
+        outcome = OVERALL_COMPLETE
+    if outcome == OVERALL_FAILED and all(
+        format_status.get(fmt) == "not_applicable" for fmt in requested
+    ):
+        outcome = REASON_OUTPUTS_NOT_APPLICABLE
+    status = _OUTCOME_TO_STATUS.get(outcome, _STATUS_FAILED)
+    return status, outcome, format_status, reasons
+
+
+# --------------------------------------------------------------------------- #
+# Runner
+# --------------------------------------------------------------------------- #
+def run_batch(
+    values: Sequence[str],
+    *,
+    out_root: Path | str,
+    settings: AppSettings | None = None,
+    formats: set[str] | None = None,
+    formats_explicit: bool = False,
+    fixture_transcript: Path | None = None,
+    fixture_content: Path | None = None,
+    doc_language: str | None = None,
+    with_reviewer_scaffolding: bool = False,
+    resume: bool = False,
+    refresh: bool = False,
+    legacy_pipeline_factory: Callable[..., Any] | None = None,
+    v2_pipeline_factory: Callable[..., Any] | None = None,
+    redirect_resolver: Callable[[str], str] | None = None,
+    doc_verifier: Callable[[Path], bool] | None = None,
+    echo: Callable[[str], None] = print,
+) -> BatchReport:
+    """Process every input once, per-source isolated, and return the report."""
+    settings = settings or AppSettings.load()
+    out_root = Path(out_root)
+    out_root.mkdir(parents=True, exist_ok=True)
+    report = BatchReport(out_root=out_root)
+
+    legacy_dirs = legacy_output_dirs(out_root, redirect_resolver=redirect_resolver) if (
+        resume or refresh
+    ) else {}
+    pipelines: dict[str, Any] = {}
+
+    def _pipeline(kind: str) -> Any:
+        if kind not in pipelines:
+            factory = legacy_pipeline_factory if kind == "v1" else v2_pipeline_factory
+            if factory is None:
+                if kind == "v1":
+                    from ytk.pipeline.orchestrator import build_default_pipeline
+
+                    factory = build_default_pipeline
+                else:
+                    from ytk.pipeline.v2 import build_v2_pipeline
+
+                    factory = build_v2_pipeline
+            pipelines[kind] = factory(settings=settings)
+        return pipelines[kind]
+
+    seen_sources: dict[str, SourceResult] = {}
+    for index, raw_value in enumerate(values, start=1):
+        value = raw_value.strip()
+        if not value:
+            continue
+        result = SourceResult(index=index, input_value=redact_text(value))
+        started = time.monotonic()
+        try:
+            _process_one(
+                result,
+                value,
+                out_root=out_root,
+                settings=settings,
+                formats=formats,
+                formats_explicit=formats_explicit,
+                fixture_transcript=fixture_transcript,
+                fixture_content=fixture_content,
+                doc_language=doc_language,
+                with_reviewer_scaffolding=with_reviewer_scaffolding,
+                resume=resume,
+                refresh=refresh,
+                legacy_dirs=legacy_dirs,
+                pipeline_getter=_pipeline,
+                redirect_resolver=redirect_resolver,
+                doc_verifier=doc_verifier,
+                seen_sources=seen_sources,
+                echo=echo,
+            )
+        except Exception as exc:  # noqa: BLE001 - one source must never abort the batch
+            _record_failure(result, exc)
+        finally:
+            result.duration_ms = int((time.monotonic() - started) * 1000)
+            report.results.append(result)
+            for dedupe_key in (
+                result.identity_key,
+                result.source_id,
+                result.acquired_source_id,
+            ):
+                if dedupe_key:
+                    seen_sources.setdefault(dedupe_key, result)
+
+    report.finished_at = time.time()
+    _write_alias_map(out_root, report)
+    _write_batch_report(out_root, report)
+    return report
+
+
+def _process_one(
+    result: SourceResult,
+    value: str,
+    *,
+    out_root: Path,
+    settings: AppSettings,
+    formats: set[str] | None,
+    formats_explicit: bool,
+    fixture_transcript: Path | None,
+    fixture_content: Path | None,
+    doc_language: str | None,
+    with_reviewer_scaffolding: bool,
+    resume: bool,
+    refresh: bool,
+    legacy_dirs: Mapping[str, Path],
+    pipeline_getter: Callable[[str], Any],
+    redirect_resolver: Callable[[str], str] | None,
+    doc_verifier: Callable[[Path], bool] | None,
+    seen_sources: MutableMapping[str, SourceResult],
+    echo: Callable[[str], None],
+) -> None:
+    try:
+        ref = resolve_batch_identity(value, redirect_resolver=redirect_resolver)
+    except (UserFacingError, ValueError) as exc:
+        result.status = _STATUS_FAILED
+        result.error_code = "IDENTITY_RESOLUTION_FAILED"
+        result.error = redact_text(str(exc))
+        return
+    result.source_id = ref.source_id
+    result.identity_key = batch_identity_key(ref)
+    result.platform = ref.platform
+
+    if not settings.is_platform_enabled(ref.platform):
+        result.status = _STATUS_BLOCKED
+        result.error_code = "PLATFORM_DISABLED"
+        result.error = f"platform {ref.platform!r} is disabled in configuration"
+        return
+
+    if result.identity_key in seen_sources:
+        duplicate = seen_sources[result.identity_key]
+        result.status = _STATUS_SKIPPED
+        result.error_code = "DUPLICATE_SOURCE"
+        result.error = (
+            f"duplicate of {duplicate.source_id or result.identity_key} (canonicalized)"
+        )
+        result.output_dir = duplicate.output_dir
+        return
+
+    directory = legacy_dirs.get(ref.source_id)
+    if directory is None:
+        directory = stable_output_dir(out_root, ref)
+    directory.mkdir(parents=True, exist_ok=True)
+    result.output_dir = str(directory)
+
+    requested = requested_formats_for(ref, formats=formats, formats_explicit=formats_explicit)
+    result.requested_formats = sorted(requested)
+
+    to_run, skip_status, skip_outcome, resume_reason = _plan_formats(
+        directory,
+        ref,
+        requested=requested,
+        resume=resume,
+        refresh=refresh,
+        doc_verifier=doc_verifier,
+        settings=settings,
+        doc_language=doc_language,
+    )
+    result.resume_reason = resume_reason
+    if resume_reason is not None and to_run:
+        echo(
+            f"[{result.index}] resume invalidated ({resume_reason}): {ref.source_id}"
+            f" — re-running {','.join(sorted(to_run))}"
+        )
+    if skip_status is not None and not to_run:
+        result.status = skip_status
+        result.outcome = skip_outcome
+        result.resumed = True
+        result.formats = _terminal_statuses(directory, sorted(requested)) or {
+            fmt: "complete" for fmt in requested
+        }
+        if skip_status in _FAILED_STATUSES:
+            result.error_code = skip_outcome or (
+                "OUTPUT_BLOCKED" if skip_status == _STATUS_BLOCKED else "OUTPUT_FAILED"
+            )
+            result.error = skip_outcome
+        echo(f"[{result.index}] resume skip ({skip_status}): {ref.source_id}")
+        return
+
+    kind = _pipeline_kind(ref, fixture_content=fixture_content)
+    pipeline = pipeline_getter(kind)
+    try:
+        run_result = _call_pipeline(
+            pipeline,
+            value=value,
+            output_dir=directory,
+            formats=to_run or requested,
+            fixture_transcript=fixture_transcript,
+            fixture_content=fixture_content,
+            doc_language=doc_language,
+            with_reviewer_scaffolding=with_reviewer_scaffolding,
+            refresh=refresh,
+            ref=ref,
+            preserve_existing_formats=resume and not refresh,
+        )
+    except (UserFacingError, ValueError) as exc:
+        _record_failure(result, exc)
+        return
+
+    if kind == "v1":
+        if doc_verifier is not None and "doc" in requested and not doc_verifier(directory):
+            raise RuntimeError(
+                "knowledge document is FALLBACK or unverified; batch completion not recorded"
+            )
+        outcome = OVERALL_COMPLETE
+        format_status = {fmt: "complete" for fmt in requested}
+        manifest = None
+    else:
+        status, outcome, format_status, reasons = _interpret_v2_result(
+            run_result, sorted(requested), directory
+        )
+        manifest = _manifest_for(directory)
+        result.formats = format_status
+        result.outcome = outcome
+        result.status = status
+        _reconcile_acquired_identity(result, ref, manifest, seen_sources)
+        if status in _FAILED_STATUSES:
+            result.error_code = outcome if outcome == REASON_OUTPUTS_NOT_APPLICABLE else (
+                "OUTPUT_FAILED" if status == _STATUS_FAILED else "OUTPUT_BLOCKED"
+            )
+            result.error = "; ".join(reasons[:3]) or None
+            return
+        write_batch_marker(
+            directory,
+            ref=ref,
+            requested=sorted(requested),
+            format_status=format_status,
+            outcome=outcome,
+            manifest=manifest,
+        )
+        return
+
+    # v1 legacy success path
+    result.status = _STATUS_COMPLETE
+    result.outcome = outcome
+    result.formats = format_status
+    write_batch_marker(
+        directory,
+        ref=ref,
+        requested=sorted(requested),
+        format_status=format_status,
+        outcome=outcome,
+        manifest=None,
+    )
+
+
+def _terminal_statuses(directory: Path, requested: Sequence[str]) -> dict[str, str]:
+    manifest = _manifest_for(directory)
+    if manifest is None:
+        return {}
+    return {
+        fmt: manifest.outputs[fmt].status
+        for fmt in requested
+        if fmt in manifest.outputs
+    }
+
+
+def _marker_is_manifest_backed(marker: Mapping[str, Any] | None) -> bool:
+    """True when the marker claims to be derived from a written manifest."""
+    if not marker:
+        return False
+    return marker.get("verified") is True or marker.get("manifest") == MANIFEST_FILENAME
+
+
+def _completion_problem(
+    store: ArtifactStore, manifest: ManifestV2, requested: Sequence[str]
+) -> str | None:
+    """A reason code when a recorded ``complete`` output no longer verifies.
+
+    Manifest entries are the completion authority; a digest/validator mismatch
+    means the artifact was corrupted or replaced after the run (A27) and the
+    requested formats must be re-run instead of trusted (R-03).
+    """
+    for fmt in requested:
+        entry = manifest.outputs.get(fmt)
+        if entry is not None and entry.status == "complete":
+            if not store.output_valid(fmt, validator_version=V2_VALIDATOR_VERSION):
+                return RESUME_REASON_STALE_MARKER
+    return None
+
+
+def _plan_formats(
+    directory: Path,
+    ref: ContentRef,
+    *,
+    requested: set[str],
+    resume: bool,
+    refresh: bool,
+    doc_verifier: Callable[[Path], bool] | None,
+    settings: AppSettings | None = None,
+    doc_language: str | None = None,
+) -> tuple[set[str], str | None, str | None, str | None]:
+    """Decide what to run and whether the whole source can be skipped.
+
+    Returns ``(formats_to_run, skip_status, skip_outcome, resume_reason)``.
+
+    The manifest is the only completion authority for v2 sources: a derived
+    ``.batch-complete.json`` marker without a readable, digest-verified manifest
+    is *not* proof of completion (R-03/A18/A27).  Legacy v1 ``video-NNN``
+    directories keep their historical marker strategy (plan §9.1).
+    """
+    store = ArtifactStore(directory)
+    manifest = store.read_manifest()
+    marker = read_batch_marker(directory)
+
+    if manifest is not None:
+        from ytk.pipeline.state import unresolved_remote_tasks
+
+        unresolved = unresolved_remote_tasks(manifest)
+        resumable = {
+            provider: record for provider, record in unresolved.items()
+            if provider == "dtk" and record.get("task_id") and ref.platform == "douyin"
+        }
+        unresolved = {
+            provider: record for provider, record in unresolved.items()
+            if provider not in resumable
+        }
+        if unresolved:
+            providers = ", ".join(sorted(unresolved))
+            return set(), _STATUS_BLOCKED, f"REMOTE_SUBMISSION_UNKNOWN for {providers}", None
+        if resumable:
+            return set(requested), None, None, None
+        if not resume and not refresh:
+            return set(requested), None, None, None
+        if refresh:
+            # Re-run everything requested; never treat old outputs as final.
+            return set(requested), None, None, None
+        content_cache = getattr(manifest, "content_cache", None)
+        if settings is not None and isinstance(content_cache, dict):
+            from ytk.pipeline.v2 import MultiPlatformPipeline
+
+            if content_cache.get("processing_fingerprint") != MultiPlatformPipeline(settings=settings)._processing_fingerprint():
+                return set(requested), None, None, "PROCESSING_SETTINGS_CHANGED"
+        problem = _completion_problem(store, manifest, sorted(requested))
+        if problem is not None:
+            # A recorded complete artifact no longer matches its digest/validator:
+            # re-run the requested formats, never resume-skip a tampered output.
+            return set(requested), None, None, problem
+        covered = _manifest_covered_formats(store, manifest, sorted(requested))
+        if settings is not None:
+            from ytk.pipeline.v2 import render_request_fingerprint
+
+            recorded = getattr(manifest, "render_fingerprints", {})
+            covered = {
+                fmt for fmt in covered
+                if fmt == "json" or manifest.outputs[fmt].status != "complete"
+                or recorded.get(fmt) == render_request_fingerprint(fmt, settings, doc_language, ref.platform)
+            }
+        missing = set(requested) - covered
+        if missing:
+            # A pipeline-recorded terminal judgment (NO_KNOWLEDGE / all formats
+            # not applicable) is reused while its validator version is current:
+            # re-dispatching a deterministic judgment would be "unconditional
+            # retry" (A30) and can be expensive (paid doc generation).
+            terminal = _manifest_terminal_judgment(manifest, sorted(requested))
+            if terminal is not None:
+                return set(), terminal[0], terminal[1], None
+            return missing, None, None, None
+        status, outcome = _status_from_manifest(manifest, sorted(requested))
+        return set(), status, outcome, None
+
+    if not resume and not refresh:
+        return set(requested), None, None, None
+    marker_matches = marker is not None and marker_source_id(marker) == ref.source_id
+    if _requires_manifest(ref, marker):
+        if marker is None:
+            # No manifest and no derived marker: a fresh (or already wiped) run.
+            return set(requested), None, None, None
+        # The marker says "complete" but the manifest that is its only authority
+        # is missing/corrupt → the derived completion cannot be trusted.
+        return set(requested), None, None, RESUME_REASON_MANIFEST_MISSING
+    if marker is not None and marker_matches:
+        if refresh:
+            return set(requested), None, None, None
+        covered = _legacy_marker_covered(
+            marker, requested, directory, doc_verifier=doc_verifier
+        )
+        missing = set(requested) - covered
+        if missing:
+            return missing, None, None, None
+        outcome = str(marker.get("outcome") or OVERALL_COMPLETE).upper()
+        status = _OUTCOME_TO_STATUS.get(outcome, _STATUS_COMPLETE)
+        return set(), status, outcome, None
+    return set(requested), None, None, None
+
+
+def _requires_manifest(ref: ContentRef, marker: Mapping[str, Any] | None) -> bool:
+    """v2-managed sources must never be completed by a derived marker alone."""
+    return ref.platform != "youtube" or _marker_is_manifest_backed(marker)
+
+
+def _pipeline_kind(ref: ContentRef, *, fixture_content: Path | None) -> str:
+    if fixture_content is not None:
+        return "v2"
+    return "v1" if ref.platform == "youtube" else "v2"
+
+
+def _record_failure(result: SourceResult, exc: BaseException) -> None:
+    from ytk.pipeline.retry import classify_error
+
+    info = classify_error(exc)
+    result.status = _STATUS_BLOCKED if info.category in _BLOCKED_CATEGORIES else _STATUS_FAILED
+    result.error_code = info.reason_code
+    result.outcome = info.category.upper()
+    result.error = redact_text(info.message)[:400]
+
+
+def _acquired_source_id(manifest: ManifestV2, ref: ContentRef) -> str | None:
+    """The authoritative source id the pipeline acquired for this input (if any)."""
+    prefix = f"{ref.platform}:{ref.content_id}"
+    candidates = [
+        source_id
+        for source_id in manifest.source_ids
+        if source_id == prefix or source_id.startswith(prefix + ":")
+    ]
+    if not candidates:
+        return None
+    variants = [source_id for source_id in candidates if source_id.startswith(prefix + ":")]
+    return variants[0] if variants else candidates[0]
+
+
+def _reconcile_acquired_identity(
+    result: SourceResult,
+    ref: ContentRef,
+    manifest: ManifestV2 | None,
+    seen_sources: MutableMapping[str, SourceResult],
+) -> None:
+    """Bind a post-acquisition variant (Bilibili ``cid``) to the same batch source.
+
+    The stable directory was computed before acquisition (part-isolated, R-08);
+    the pipeline may report a more specific ``source_id`` afterwards.  That is
+    the *same* source — register it so a later duplicate input is not processed
+    twice, but never move or rename the part-isolated directory.
+    """
+    if manifest is None:
+        return
+    acquired = _acquired_source_id(manifest, ref)
+    if acquired is None:
+        return
+    result.acquired_source_id = acquired
+    seen_sources.setdefault(acquired, result)
+
+
+def _write_alias_map(out_root: Path, report: BatchReport) -> Path | None:
+    aliases = [
+        {
+            "input": result.input_value,
+            "source_id": result.source_id,
+            "identity_key": result.identity_key,
+            "acquired_source_id": result.acquired_source_id,
+            "platform": result.platform,
+            "output_dir": (
+                str(Path(result.output_dir).relative_to(out_root))
+                if result.output_dir
+                else None
+            ),
+        }
+        for result in report.results
+        if result.source_id
+    ]
+    try:
+        return ArtifactStore(out_root).write_json(
+            BATCH_ALIASES_FILENAME, {"schema": 1, "aliases": aliases}
+        )
+    except Exception as exc:  # noqa: BLE001 - alias bookkeeping must not fail a batch
+        LOGGER.warning("could not write batch alias map: %s", exc)
+        return None
+
+
+def _write_batch_report(out_root: Path, report: BatchReport) -> Path | None:
+    try:
+        return ArtifactStore(out_root).write_json(BATCH_REPORT_FILENAME, report.to_dict())
+    except Exception as exc:  # noqa: BLE001
+        LOGGER.warning("could not write batch report: %s", exc)
+        return None
+
+
+__all__ = [
+    "BATCH_ALIASES_FILENAME",
+    "BATCH_MARKER_FILENAME",
+    "BATCH_MARKER_SCHEMA",
+    "BATCH_REPORT_FILENAME",
+    "BatchReport",
+    "RESUME_REASON_MANIFEST_MISSING",
+    "RESUME_REASON_STALE_MARKER",
+    "SourceResult",
+    "V2_VALIDATOR_VERSION",
+    "batch_identity_key",
+    "default_formats_for",
+    "display_part",
+    "legacy_output_dirs",
+    "marker_source_id",
+    "read_batch_marker",
+    "requested_formats_for",
+    "resolve_batch_identity",
+    "run_batch",
+    "stable_output_dir",
+    "stable_source_dirname",
+    "write_batch_marker",
+]