openmd-cli 0.1.0__py3-none-any.whl

openmd/core/planner.py

@@ -0,0 +1,345 @@
+"""Pure V1 planning projection. This module must never construct a provider."""
+
+from __future__ import annotations
+
+import re
+import stat
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+from urllib.parse import unquote, urlparse
+
+from openmd.core.detector import DOCUMENT_EXTENSIONS, IMAGE_TEXT_EXTENSIONS, detect_platform
+from openmd.backends.bilibili import (
+    BILIBILI_OPENCLI_DISCOVERY_BACKEND,
+    BILIBILI_OPENCLI_SUBTITLE_BACKEND,
+    extract_bilibili_uid,
+)
+from openmd.backends.douyin import DOUYIN_OPENMD_CREATOR_VIDEOS_BACKEND, douyin_user_target
+from openmd.backends.web_article import OPENCLI_WEB_READ_BACKEND
+from openmd.backends.youtube import OPENCLI_YOUTUBE_TRANSCRIPT_BACKEND
+from openmd.backends.youtube_transcript import extract_youtube_video_id
+from openmd.backends.youtube_discovery import (
+    YOUTUBE_DATA_API_PLAYLIST_DISCOVERY_BACKEND,
+    YOUTUBE_DISCOVERY_BACKEND,
+    YOUTUBE_PLAYLIST_DISCOVERY_BACKEND,
+    normalize_youtube_collection_url,
+    youtube_data_api_channel_lookup,
+    youtube_data_api_playlist_id,
+)
+
+from .capabilities import CapabilityRecord, get_capability
+from .operation_contract import COLLECTION_BINDINGS, collect_all_allowed
+
+
+READ_ONLY_SIDE_EFFECTS = (
+    "no_provider_execution",
+    "no_network",
+    "no_browser_or_session_access",
+    "no_input_content_read",
+    "no_media_or_model_download",
+)
+_WINDOWS_PATH = re.compile(r"^[A-Za-z]:[\\/]")
+_AUDIO_SUFFIXES = {".mp3", ".m4a", ".wav", ".flac"}
+_VIDEO_SUFFIXES = {".mp4", ".m4v", ".mov", ".webm", ".mkv"}
+
+
+@dataclass(frozen=True)
+class PlanResult:
+    ok: bool
+    detected_source: str
+    input_kind: str
+    capability_tier: str | None
+    implementation_state: str | None
+    source_exhausted_requested: bool
+    source_exhausted_approved: bool
+    planned_provider: str | None
+    runtime_provider: str | None
+    binding_state: str | None
+    execution_ready: bool
+    required_gates: tuple[str, ...]
+    cost_class: str | None
+    side_effects: tuple[str, ...]
+    blocked_reason: str | None
+    error_code: str | None
+    discovery_provider: str | None = None
+    required_backend: str | None = None
+    required_discovery_backend: str | None = None
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "ok": self.ok,
+            "detected_source": self.detected_source,
+            "input_kind": self.input_kind,
+            "capability_tier": self.capability_tier,
+            "implementation_state": self.implementation_state,
+            "source_exhausted_requested": self.source_exhausted_requested,
+            "source_exhausted_approved": self.source_exhausted_approved,
+            "planned_provider": self.planned_provider,
+            "runtime_provider": self.runtime_provider,
+            "binding_state": self.binding_state,
+            "execution_ready": self.execution_ready,
+            "required_gates": list(self.required_gates),
+            "cost_class": self.cost_class,
+            "side_effects": list(self.side_effects),
+            "blocked_reason": self.blocked_reason,
+            "error_code": self.error_code,
+            "discovery_provider": self.discovery_provider,
+            "required_backend": self.required_backend,
+            "required_discovery_backend": self.required_discovery_backend,
+            "readiness_scope": "static_contract_only",
+            "runtime_checks": "dependencies, models, credentials and source access are not probed",
+        }
+
+
+def _local_candidate(value: str) -> Path | None:
+    """Resolve spelling only. Never expand a folder or touch an input body."""
+    if _WINDOWS_PATH.match(value):
+        return Path(value)
+    parsed = urlparse(value)
+    if parsed.scheme == "file":
+        if parsed.netloc not in {"", "localhost"}:
+            raise ValueError("network_path_not_supported")
+        raw = unquote(parsed.path)
+        if raw.startswith(("\\\\", "//")):
+            raise ValueError("network_path_not_supported")
+        if re.match(r"^/[A-Za-z]:/", raw):
+            raw = raw[1:]
+        return Path(raw)
+    if parsed.scheme:
+        return None
+    if value.startswith(("\\\\", "//")):
+        raise ValueError("network_path_not_supported")
+    return Path(value) if value else None
+
+
+def _local_source(path: Path) -> str:
+    suffix = path.suffix.lower()
+    if suffix in _AUDIO_SUFFIXES:
+        return "local_audio"
+    if suffix in _VIDEO_SUFFIXES:
+        return "local_video"
+    if suffix in DOCUMENT_EXTENSIONS:
+        return "pdf_office"
+    if suffix in IMAGE_TEXT_EXTENSIONS:
+        return "screenshot"
+    return "unknown"
+
+
+def _inspect_input(value: str) -> tuple[str, str, Path | None, str | None]:
+    """Return source/shape/path/error using URL syntax or one local stat only."""
+    if re.fullmatch(r"BV[0-9A-Za-z]+", value):
+        return "bilibili", "single", None, None
+    try:
+        path = _local_candidate(value)
+        if path is not None:
+            source = _local_source(path)
+            try:
+                mode = path.stat().st_mode
+            except FileNotFoundError:
+                return source, "file", path, "input_missing" if source != "unknown" else "source_unsupported"
+            except OSError:
+                return source, "file", path, "input_stat_failed"
+            if stat.S_ISDIR(mode):
+                return "unknown", "folder", path, "folder_source_not_inspected"
+            if not stat.S_ISREG(mode):
+                return source, "file", path, "input_not_regular_file"
+            return source, "file", path, None
+        parsed = urlparse(value)
+        if parsed.scheme in {"http", "https"}:
+            if not parsed.hostname or any(c.isspace() for c in value):
+                return "unknown", "single", None, "source_unsupported"
+            # Only HTTP URLs reach the legacy detector, so its local recursive
+            # directory classification is unreachable from this projection.
+            source = detect_platform(value)
+            if source == "pdf":
+                source = "pdf_office"
+            if source == "xiaohongshu" and "/user/profile/" in parsed.path:
+                source = "xiaohongshu_creator"
+            return source, _infer_remote_kind(value, source), None, None
+        if parsed.scheme in {"weixin", "wx-official", "wechat-official", "wechat", "wx", "wx-group", "wechat-group", "wx-moments", "wechat-moments", "qq", "qq-group", "qq_export", "qq-export"}:
+            source = detect_platform(value)
+            if source != "web_article":
+                return source, "collection", None, None
+    except (ValueError, OSError):
+        return "unknown", "single", None, "source_unsupported"
+    return "unknown", "single", None, "source_unsupported"
+
+
+def _infer_remote_kind(value: str, source: str) -> str:
+    parsed = urlparse(value)
+    path = parsed.path.lower()
+    if source == "youtube" and normalize_youtube_collection_url(value) is not None:
+        return "collection"
+    if source == "bilibili" and ((parsed.hostname or "").startswith("space.") or "/collection" in path or "/lists/" in path):
+        return "collection"
+    if source == "xiaoyuzhou" and "/podcast/" in path:
+        return "collection"
+    if source == "douyin" and ("/user/" in path or "sec_uid=" in parsed.query):
+        return "collection"
+    if source in {"weixin", "xiaoe"} and any(marker in path for marker in ("/author", "/collection", "/course", "/catalog", "/column")):
+        return "collection"
+    if source in {"xiaohongshu_creator", "wechat_group", "wechat_moments", "qq_group"}:
+        return "collection"
+    return "single"
+
+
+def _blocked(
+    source_id: str,
+    input_kind: str,
+    request_all: bool,
+    *,
+    code: str,
+    reason: str,
+    record: CapabilityRecord | None = None,
+) -> PlanResult:
+    return PlanResult(
+        ok=False,
+        detected_source=source_id,
+        input_kind=input_kind,
+        capability_tier=record.tier if record else None,
+        implementation_state=record.state if record else None,
+        source_exhausted_requested=request_all,
+        source_exhausted_approved=False,
+        planned_provider=record.planned_provider if record else None,
+        runtime_provider=record.runtime_provider if record else None,
+        binding_state=record.binding_state if record else None,
+        execution_ready=False,
+        required_gates=record.required_gates if record else (),
+        cost_class=record.cost_class if record else None,
+        side_effects=READ_ONLY_SIDE_EFFECTS,
+        blocked_reason=reason,
+        error_code=code,
+    )
+
+
+def build_plan(
+    input_value: str,
+    *,
+    input_kind: str = "auto",
+    request_all: bool = False,
+    available_gates: set[str] | None = None,
+) -> PlanResult:
+    value = input_value.strip()
+    source_id, inferred_kind, local_path, input_error = _inspect_input(value)
+    selected_kind = inferred_kind if input_kind == "auto" else input_kind
+    if input_error:
+        reasons = {
+            "input_missing": "local input file does not exist",
+            "input_stat_failed": "local input metadata could not be read",
+            "input_not_regular_file": "local input is not a regular file",
+            "folder_source_not_inspected": "folder contents are not inspected by plan; plan individual files",
+        }
+        return _blocked(source_id, selected_kind, request_all, code=input_error, reason=reasons.get(input_error, "input source is not approved"))
+    try:
+        record = get_capability(source_id)
+    except ValueError:
+        return _blocked(source_id, selected_kind, request_all, code="source_unsupported", reason="input source is not approved")
+
+    if local_path is not None and selected_kind != inferred_kind:
+        return _blocked(source_id, selected_kind, request_all, code="input_shape_unsupported", reason="input kind does not match local input", record=record)
+    if inferred_kind == "collection" and selected_kind == "single":
+        return _blocked(source_id, selected_kind, request_all, code="input_shape_unsupported", reason="collection input cannot be planned as a single item", record=record)
+    if selected_kind not in record.input_kinds:
+        return _blocked(source_id, selected_kind, request_all, code="input_shape_unsupported", reason="input kind is not approved for this source", record=record)
+    if record.state == "paused":
+        return _blocked(source_id, selected_kind, request_all, code="capability_paused", reason="source is paused before provider execution", record=record)
+    if request_all and selected_kind not in record.source_exhausted_kinds:
+        return _blocked(source_id, selected_kind, True, code="all_not_approved", reason="source exhaustion is not approved for this input kind", record=record)
+
+    gates = available_gates or set()
+    required = list(record.required_gates)
+    runtime_provider = record.runtime_provider
+    if not set(required) <= gates and source_id in {"youtube", "bilibili", "weixin", "web_article"}:
+        # Project an allowed fallback from the same declarations as execution.
+        # Planning stays static: do not construct or probe a provider.
+        from openmd.routing.loader import effective_routes, load_catalog
+
+        catalog = load_catalog()
+        for route in effective_routes(source_id):
+            candidate = next((backend for backend in route["chain"]
+                              if set(catalog.capabilities[backend].get("gates", [])) <= gates), None)
+            if candidate:
+                runtime_provider = candidate
+                required = list(catalog.capabilities[candidate].get("gates", []))
+                break
+    discovery_provider = None
+    required_discovery_backend = None
+    operation_block = None
+    if selected_kind == "collection":
+        binding = COLLECTION_BINDINGS.get(source_id)
+        if binding is None:
+            operation_block = "collection discovery is not implemented"
+        else:
+            discovery_provider, discovery_gates = binding
+            if source_id == "bilibili" and "network" not in gates and "browser" in gates:
+                discovery_provider = BILIBILI_OPENCLI_DISCOVERY_BACKEND
+                discovery_gates = ("browser",)
+            if source_id == "bilibili" and extract_bilibili_uid(value) is None:
+                operation_block = "only Bilibili creator discovery is currently implemented"
+            if source_id == "douyin" and not request_all and douyin_user_target(value) is None:
+                operation_block = "bounded Douyin discovery requires a creator identifier"
+            if source_id == "youtube":
+                playlist = youtube_data_api_playlist_id(value) is not None
+                if normalize_youtube_collection_url(value) is None:
+                    operation_block = "input is not a supported YouTube collection"
+                if "browser" in gates and ("network" not in gates or "account" not in gates):
+                    discovery_provider = YOUTUBE_PLAYLIST_DISCOVERY_BACKEND if playlist else YOUTUBE_DISCOVERY_BACKEND
+                    discovery_gates = ("browser",)
+                elif playlist:
+                    discovery_provider = YOUTUBE_DATA_API_PLAYLIST_DISCOVERY_BACKEND
+                elif youtube_data_api_channel_lookup(value) is None:
+                    operation_block = "this YouTube collection requires the browser discovery gate"
+                required_discovery_backend = discovery_provider
+            required.extend(discovery_gates)
+            if request_all and source_id == "douyin":
+                discovery_provider = DOUYIN_OPENMD_CREATOR_VIDEOS_BACKEND
+                required_discovery_backend = discovery_provider
+    elif selected_kind == "list":
+        operation_block = "list contents are not inspected by plan; plan individual inputs"
+    elif source_id == "douyin":
+        operation_block = "standalone Douyin input resolution is not implemented; use creator collection"
+    elif source_id == "youtube" and extract_youtube_video_id(value) is None:
+        operation_block = "input does not identify a YouTube video"
+    if request_all and not collect_all_allowed(source_id, browser_enabled=True, network_enabled=True, discovery_provider=discovery_provider):
+        operation_block = "source exhaustion is not implemented for this operation"
+    if local_path is None and source_id in {"pdf_office", "screenshot"}:
+        operation_block = "this provider requires a local file"
+    missing_gates = tuple(dict.fromkeys(gate for gate in required if gate not in gates))
+    execution_ready = record.binding_state == "implemented" and not missing_gates and record.state not in {"blocked", "paused"} and operation_block is None
+    if operation_block:
+        blocked_reason = operation_block
+    elif record.binding_state == "unimplemented":
+        blocked_reason = "runtime provider is not implemented"
+    elif missing_gates:
+        blocked_reason = f"missing required gate(s): {', '.join(missing_gates)}"
+    elif record.state == "blocked":
+        blocked_reason = "capability is blocked"
+    else:
+        blocked_reason = None
+    # An executable route can differ from the default CLI primary. Surface the
+    # exact override so callers never silently execute a placeholder instead.
+    required_backend = None
+    if source_id in {"local_audio", "local_video", "tutorial_video", "douyin", "youtube", "bilibili", "weixin", "web_article"} or runtime_provider != record.planned_provider:
+        required_backend = runtime_provider
+    return PlanResult(
+        ok=True,
+        detected_source=source_id,
+        input_kind=selected_kind,
+        capability_tier=record.tier,
+        implementation_state=record.state,
+        source_exhausted_requested=request_all,
+        source_exhausted_approved=request_all and selected_kind in record.source_exhausted_kinds,
+        planned_provider=record.planned_provider,
+        runtime_provider=runtime_provider,
+        binding_state=record.binding_state,
+        execution_ready=execution_ready,
+        required_gates=missing_gates,
+        cost_class=record.cost_class,
+        side_effects=READ_ONLY_SIDE_EFFECTS,
+        blocked_reason=blocked_reason,
+        error_code=None,
+        discovery_provider=discovery_provider,
+        required_backend=required_backend,
+        required_discovery_backend=required_discovery_backend,
+    )

openmd/core/preferences.py

@@ -0,0 +1,61 @@
+"""Nonsecret preferences for the interactive single-link workflow."""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from uuid import uuid4
+
+
+SCHEMA = "openmd.preferences/v1"
+
+
+def data_directory() -> Path:
+    configured = os.environ.get("OPENMD_DATA_DIR")
+    return Path(configured).expanduser().resolve() if configured else Path.home() / ".openmd"
+
+
+def preferences_path() -> Path:
+    return data_directory() / "preferences.json"
+
+
+def load_preferences() -> dict[str, object]:
+    """Malformed preferences never grant permissions or block extraction."""
+    path = preferences_path()
+    try:
+        if not path.is_file() or path.stat().st_size > 16 * 1024:
+            return {}
+        value = json.loads(path.read_text(encoding="utf-8-sig"))
+    except (OSError, ValueError):
+        return {}
+    if not isinstance(value, dict) or value.get("schema") != SCHEMA:
+        return {}
+    result: dict[str, object] = {}
+    directory = value.get("last_save_directory")
+    if isinstance(directory, str) and directory.strip():
+        result["last_save_directory"] = directory
+    if type(value.get("browser_always")) is bool:
+        result["browser_always"] = value["browser_always"]
+    return result
+
+
+def save_preferences(*, last_save_directory: Path | None = None, browser_always: bool | None = None) -> None:
+    """Only these explicit nonsecret values can enter the preference file."""
+    value = {"schema": SCHEMA, **load_preferences()}
+    if last_save_directory is not None:
+        value["last_save_directory"] = str(last_save_directory.expanduser().resolve())
+    if browser_always is not None:
+        value["browser_always"] = bool(browser_always)
+    path = preferences_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_name(f".{path.name}.{uuid4().hex}.tmp")
+    try:
+        with temporary.open("x", encoding="utf-8", newline="\n") as handle:
+            json.dump(value, handle, ensure_ascii=False, indent=2)
+            handle.write("\n")
+            handle.flush()
+            os.fsync(handle.fileno())
+        temporary.replace(path)
+    finally:
+        temporary.unlink(missing_ok=True)

openmd/core/readable.py

@@ -0,0 +1,129 @@
+"""Conservative display formatting for caption text, without rewriting words."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+
+_TIME = r"\d{1,}:[0-5]\d(?::[0-5]\d)?"
+# Local ASR emits "[start-end] text"; a caption keeps only its start time for display.
+_CAPTION = re.compile(rf"^\[(?P<time>{_TIME})(?:-{_TIME})?\][ \t]+(?P<text>.+)$")
+_TABLE_ROW = re.compile(
+    rf"^\s*\|\s*(?P<time>{_TIME})\s*\|(?P<speaker>[^|]*)\|(?P<text>.*)\|\s*$"
+)
+_TABLE_HEADER = re.compile(r"^\s*\|\s*timestamp\s*\|\s*speaker\s*\|\s*text\s*\|\s*$", re.I)
+_TABLE_SEPARATOR = re.compile(r"^\s*\|\s*:?-+:?\s*\|\s*:?-+:?\s*\|\s*:?-+:?\s*\|\s*$")
+_MARKDOWN_SPECIAL = re.compile(r"([\\`*_{}\[\]<>#!|])")
+_SENTENCE_END = re.compile(r"[.!?。！？][\"'’”）)]*$")
+
+
+def markdown_label(value: object) -> str:
+    """Escape external text used in a generated heading or label."""
+    text = str(value or "").replace("\r", " ").replace("\n", " ")
+    return _MARKDOWN_SPECIAL.sub(r"\\\1", text)
+
+
+@dataclass(frozen=True)
+class _Caption:
+    timestamp: str
+    text: str
+    speaker: str = ""
+    chapter: bool = False
+
+    @property
+    def seconds(self) -> int:
+        total = 0
+        for part in self.timestamp.split(":"):
+            total = total * 60 + int(part)
+        return total
+
+
+def _table_captions(text: str) -> list[_Caption] | None:
+    lines = [line for line in text.splitlines() if line.strip()]
+    if len(lines) < 3 or not _TABLE_HEADER.fullmatch(lines[0]) or not _TABLE_SEPARATOR.fullmatch(lines[1]):
+        return None
+    captions = []
+    for line in lines[2:]:
+        match = _TABLE_ROW.fullmatch(line)
+        if match is None:
+            return None
+        content = match.group("text").strip()
+        if not content:
+            return None
+        captions.append(_Caption(match.group("time"), content, match.group("speaker").strip(), content.startswith("[Chapter] ")))
+    return captions
+
+
+def _line_captions(text: str) -> list[_Caption] | None:
+    captions = []
+    for line in text.splitlines():
+        if not line.strip():
+            continue
+        match = _CAPTION.fullmatch(line)
+        if match is None:
+            return None
+        captions.append(_Caption(match.group("time"), match.group("text")))
+    return captions or None
+
+
+def format_transcript(text: str, *, source_type: str | None, platform: str | None = None) -> str:
+    """Group recognized captions only; articles and mixed/unknown formats pass through.
+
+    No text is summarized, corrected, sorted, deduplicated or repunctuated.
+    Only exact transcript table/line formats are recognized. A partly matching
+    document is returned intact rather than dropping unfamiliar source lines.
+    """
+    if source_type not in {"video", "audio"}:
+        return text
+    captions = _table_captions(text)
+    table = captions is not None
+    if captions is None:
+        captions = _line_captions(text)
+    if captions is None:
+        return text
+
+    blocks: list[str] = []
+    group: list[_Caption] = []
+    group_chars = 0
+
+    def flush() -> None:
+        if not group:
+            return
+        first = group[0]
+        label = f"{markdown_label(first.speaker)} ({first.timestamp})" if first.speaker else first.timestamp
+        blocks.append(f"**{label}**\n\n" + " ".join(caption.text for caption in group))
+
+    for caption in captions:
+        if caption.chapter:
+            flush()
+            group = []
+            group_chars = 0
+            heading = caption.text[len("[Chapter] "):]
+            blocks.append(f"## {markdown_label(heading)} ({caption.timestamp})")
+            continue
+        if group:
+            duration = caption.seconds - group[0].seconds
+            gap = caption.seconds - group[-1].seconds
+            if table:
+                # Use source timestamps to bound long turns even when captions
+                # have no sentence-ending punctuation. Never split row text.
+                boundary = (
+                    caption.speaker != group[0].speaker
+                    or duration >= 60
+                    or group_chars >= 2400
+                    or (group_chars >= 1200 and bool(_SENTENCE_END.search(group[-1].text.rstrip())))
+                )
+            else:
+                boundary = duration >= 30 or gap >= 6 or group_chars >= 450
+            if gap < 0:
+                # Retain source order, even if upstream times move backwards.
+                boundary = True
+            if boundary:
+                flush()
+                group = []
+                group_chars = 0
+        group.append(caption)
+        group_chars += len(caption.text)
+    flush()
+    return "\n\n".join(blocks)