openmd-cli 0.1.0__py3-none-any.whl

openmd/backends/generic_media.py

@@ -0,0 +1,77 @@
+"""Runtime providers for local generic media inputs."""
+
+from __future__ import annotations
+
+from openmd.adapters.local_audio.adapter import LocalAudioAdapter
+from openmd.runtime.asr import get_asr_backend
+from openmd.adapters.local_audio.asr import MissingASRBackend, MockASRBackend
+from openmd.backends.base import (
+    BackendCapability,
+    BackendDescriptor,
+    BackendExtractRequest,
+    BackendHealth,
+)
+from openmd.backends.executor import BackendProviderRegistry
+from openmd.backends.registry import BackendRuntimeOptions
+from openmd.core.models import TranscriptResult
+
+
+class LocalAudioASRProvider:
+    def __init__(self, backend) -> None:
+        self.backend = backend
+        self.name = backend.name
+
+    def descriptor(self) -> BackendDescriptor:
+        source_level = "partial_extraction" if isinstance(self.backend, MissingASRBackend) else "asr"
+        return BackendDescriptor(
+            name=self.name,
+            layer="local",
+            capabilities=(
+                BackendCapability(
+                    platforms=("generic_media",),
+                    source_types=("audio",),
+                    input_types=("single", "batch", "collection", "generic"),
+                    source_levels=(source_level,),
+                    requires_network=False,
+                    requires_credentials=False,
+                ),
+            ),
+            description="Local audio ASR provider.",
+            provider="openmd.local_audio",
+        )
+
+    def health(self) -> BackendHealth:
+        is_available = getattr(self.backend, "is_available", None)
+        if callable(is_available) and not is_available():
+            return BackendHealth(
+                ok=False,
+                error_code="asr_backend_missing",
+                error_summary=f"{self.name} is not installed or enabled.",
+            )
+        if isinstance(self.backend, MissingASRBackend):
+            return BackendHealth(
+                ok=False,
+                error_code="asr_backend_missing",
+                error_summary=f"{self.name} is not installed or enabled.",
+            )
+        return BackendHealth(ok=True, dependencies={self.name: "ok"})
+
+    def extract(self, request: BackendExtractRequest) -> TranscriptResult:
+        adapter = LocalAudioAdapter("generic_media", request.primary_backend)
+        return adapter.extract_text(
+            request.item,
+            backend=request.primary_backend,
+            backend_chain=(request.primary_backend,),
+            allow_fallback=False,
+        )
+
+
+def register_backend_providers(
+    registry: BackendProviderRegistry,
+    options: BackendRuntimeOptions,
+) -> None:
+    registry.register("local_audio.mock", LocalAudioASRProvider(MockASRBackend()))
+    registry.register("asr.faster_whisper", LocalAudioASRProvider(get_asr_backend("asr.faster_whisper")))
+    registry.register("asr.auto", LocalAudioASRProvider(get_asr_backend("asr.auto")))
+    registry.register("asr.qwen3", LocalAudioASRProvider(get_asr_backend("asr.qwen3")))
+    registry.register("asr.fallback", LocalAudioASRProvider(MissingASRBackend("asr.fallback")))

openmd/backends/interactive.py

@@ -0,0 +1,60 @@
+"""Defer daily browser consent until execution reaches a browser candidate."""
+
+from __future__ import annotations
+
+from dataclasses import replace
+
+from openmd.core.models import TranscriptResult
+from openmd.routing.loader import load_catalog
+from openmd.routing.providers import gates_open
+
+from .executor import BackendProviderRegistry
+from .registry import create_runtime_backend_provider_registry
+
+
+class DeferredBrowserRegistry(BackendProviderRegistry):
+    def __init__(self, original, platform, args, authorize):
+        super().__init__()
+        self.original = original
+        self.platform = platform
+        self.args = args
+        self.authorize = authorize
+
+    def provider_for(self, backend):
+        provider = self.original.provider_for(backend)
+        # The normal registry uses safe skeleton placeholders for closed gates.
+        from .executor import SkeletonBackendProvider
+        if provider is not None and not isinstance(provider, SkeletonBackendProvider):
+            return provider
+        capability = load_catalog().capabilities.get(backend, {})
+        if "browser" not in capability.get("gates", []):
+            return provider
+        from openmd.cli import _backend_runtime_options
+
+        possible = replace(_backend_runtime_options(self.args), enable_browser_backends=True)
+        if not gates_open(capability, possible):
+            return provider
+        return _DeferredBrowserProvider(self, backend)
+
+
+class _DeferredBrowserProvider:
+    def __init__(self, registry, backend):
+        self.registry = registry
+        self.name = backend
+
+    def extract(self, request):
+        if self.registry.authorize(self.name):
+            from openmd.cli import _backend_runtime_options
+
+            registry = create_runtime_backend_provider_registry(
+                self.registry.platform, _backend_runtime_options(self.registry.args)
+            )
+            provider = registry.provider_for(self.name)
+            if provider is not None:
+                self.registry.original = registry
+                return provider.extract(request)
+        return TranscriptResult(
+            text="", status="failed", source_level="partial_extraction", backend=self.name,
+            error_code="browser_permission_required",
+            error_summary="此路线需要浏览器读取许可；可在提示中允许一次，或显式使用 --enable-browser-backends。",
+        )

openmd/backends/ocr.py

@@ -0,0 +1,41 @@
+"""OCR providers shared by screenshot and document route chains."""
+
+from __future__ import annotations
+
+from openmd.backends.base import BackendCapability, BackendDescriptor, BackendHealth
+from openmd.ocr.pipeline import LocalOCRBackend
+
+
+class OCRRouterProvider:
+    def __init__(self, name: str, options) -> None:
+        self.name, self.options = name, options
+
+    def descriptor(self):
+        return BackendDescriptor(name=self.name, layer="local", provider="openmd.ocr", description="Serial local Ollama OCR with monitored fallback and page receipts.", capabilities=(BackendCapability(platforms=("screenshot", "pdf"), source_types=("image_text", "document"), input_types=("single", "batch", "collection", "generic"), source_levels=("ocr",), requires_network=True),))
+
+    def health(self):
+        if not self.options.enable_network_backends:
+            return BackendHealth(ok=False, error_code="ocr_permission_required", error_summary="Enable network backends explicitly to call the configured Ollama API.")
+        from openmd.ocr.config import settings
+        from openmd.ocr.ollama import probe_ollama
+
+        report = probe_ollama(settings())
+        key = "minicpm_model" if self.name == "ocr.minicpm" else "ovis_model"
+        ready = bool(report["models"].get(key, {}).get("available"))
+        return BackendHealth(ok=ready, dependencies={"ollama": str(report.get("version"))}, error_code=None if ready else report.get("error_code"))
+
+    def extract(self, request):
+        if request.item.platform == "pdf":
+            from openmd.ocr.documents import extract_pdf
+
+            return extract_pdf(request.item, backend=self.name, allow_ocr=self.options.enable_network_backends)
+        backend = LocalOCRBackend(self.name)
+        if not self.options.enable_network_backends:
+            return backend.failed("ocr_permission_required", request.item.title or "screenshot", "Use --enable-network-backends to call the configured Ollama API.")
+        source = request.item.raw_meta.get("local_path") or request.item.url
+        return backend.recognize(source, title=request.item.title or "screenshot")
+
+
+def register_ocr_providers(registry, options):
+    for name in ("ocr.ovis3", "ocr.minicpm", "ocr.chain"):
+        registry.register(name, OCRRouterProvider(name, options))

openmd/backends/opencli.py

@@ -0,0 +1,82 @@
+"""Safe OpenCLI command wrapper scaffold."""
+
+from __future__ import annotations
+
+import subprocess
+from collections.abc import Sequence
+
+from openmd.core.models import TranscriptResult
+from openmd.core.secrets import sanitize_error_summary
+
+from .base import BackendCommandResult
+
+from openmd.runtime.interpreters import run_interruptibly
+
+
+class OpenCliBackend:
+    """Run read-only OpenCLI commands without persisting raw stdout or stderr."""
+
+    def __init__(self, executable: str = "opencli", *, timeout: int = 60) -> None:
+        self.executable = executable
+        self.timeout = timeout
+
+    def run(self, args: Sequence[str], *, name: str = "opencli") -> BackendCommandResult:
+        command = (self.executable, *args)
+        try:
+            completed = run_interruptibly(
+                command,
+                capture_output=True,
+                text=True,
+                check=False,
+                timeout=self.timeout,
+            )
+        except FileNotFoundError as exc:
+            return BackendCommandResult(
+                name=name,
+                command=command,
+                exit_code=127,
+                error_summary=sanitize_error_summary(exc),
+            )
+        except subprocess.TimeoutExpired as exc:
+            return BackendCommandResult(
+                name=name,
+                command=command,
+                exit_code=124,
+                stdout_summary=sanitize_error_summary(exc.stdout or "") or "",
+                stderr_summary=sanitize_error_summary(exc.stderr or "") or "",
+                error_summary=f"command timed out after {self.timeout}s",
+            )
+
+        stderr_summary = sanitize_error_summary(completed.stderr or "") or ""
+        stdout_summary = sanitize_error_summary(completed.stdout or "") or ""
+        return BackendCommandResult(
+            name=name,
+            command=command,
+            exit_code=completed.returncode,
+            stdout_summary=stdout_summary,
+            stderr_summary=stderr_summary,
+            error_summary=stderr_summary if completed.returncode else None,
+        )
+
+
+def command_result_to_transcript(
+    result: BackendCommandResult,
+    *,
+    source_level: str = "third_party_extractor",
+) -> TranscriptResult:
+    if result.success and result.stdout_summary.strip():
+        return TranscriptResult(
+            text=result.stdout_summary,
+            status="verified",
+            source_level=source_level,  # type: ignore[arg-type]
+            backend=result.name,
+            source_notes=["OpenCLI wrapper returned summarized stdout"],
+        )
+    return TranscriptResult(
+        text="",
+        status="failed",
+        source_level="partial_extraction",
+        backend=result.name,
+        error_code=f"backend_exit_{result.exit_code}",
+        error_summary=result.error_summary or result.stderr_summary or "OpenCLI command returned no usable text.",
+    )

openmd/backends/pdf.py

@@ -0,0 +1,122 @@
+"""Runtime providers for PDF/Office inputs under the legacy PDF identity."""
+
+from __future__ import annotations
+
+from dataclasses import replace
+
+from openmd.adapters.pdf.adapter import PdfDocumentAdapter
+from openmd.adapters.pdf.backends.markitdown import MarkItDownPDFParser
+from openmd.adapters.pdf.parser import MissingPDFParser, MockPDFParser
+from openmd.backends.base import (
+    BackendCapability,
+    BackendDescriptor,
+    BackendExtractRequest,
+    BackendHealth,
+)
+from openmd.backends.executor import BackendProviderRegistry
+from openmd.backends.registry import BackendRuntimeOptions
+from openmd.core.models import TranscriptResult
+
+
+class PDFParserProvider:
+    def __init__(self, parser, options: BackendRuntimeOptions | None = None) -> None:
+        self.parser = parser
+        self.name = parser.name
+        self.options = options or BackendRuntimeOptions()
+
+    def descriptor(self) -> BackendDescriptor:
+        source_level = "native_text" if isinstance(self.parser, (MockPDFParser, MarkItDownPDFParser)) else "partial_extraction"
+        return BackendDescriptor(
+            name=self.name,
+            layer="local",
+            capabilities=(
+                BackendCapability(
+                    platforms=("pdf",),
+                    source_types=("document",),
+                    input_types=("single", "batch", "collection", "generic"),
+                    source_levels=(source_level,),
+                    requires_network=False,
+                    requires_credentials=False,
+                ),
+            ),
+            description="Local PDF/DOCX/PPTX/XLSX document parser provider.",
+            provider="openmd.pdf",
+        )
+
+    def health(self) -> BackendHealth:
+        is_available = getattr(self.parser, "is_available", None)
+        if callable(is_available) and not is_available():
+            return BackendHealth(
+                ok=False,
+                error_code="document_backend_missing",
+                error_summary=f"{self.name} is not installed or enabled.",
+            )
+        if isinstance(self.parser, MissingPDFParser):
+            return BackendHealth(
+                ok=False,
+                error_code="document_backend_missing",
+                error_summary=f"{self.name} is not installed or enabled.",
+            )
+        return BackendHealth(ok=True, dependencies={self.name: "ok"})
+
+    def extract(self, request: BackendExtractRequest) -> TranscriptResult:
+        from openmd.adapters.local_audio.adapter import local_path_from_input
+        from openmd.ocr.documents import PDFDependencyMissing, document_failure, inspect_pdf
+
+        source = request.item.raw_meta.get("local_path") or request.item.url
+        path = local_path_from_input(source)
+        probes = None
+        if self.name == "pdf.markitdown" and path and path.is_file() and path.suffix.lower() == ".pdf":
+            try:
+                probes = inspect_pdf(path)
+                if any(not page["reliable"] for page in probes):
+                    return document_failure(self.name, request.item.title or path.stem, "pdf_ocr_required", "One or more pages require OCR; continue to the page router.")
+            except PDFDependencyMissing as exc:
+                return document_failure(self.name, request.item.title or path.stem, "pymupdf_missing", str(exc))
+            except (OSError, ValueError, RuntimeError):
+                return document_failure(self.name, request.item.title or path.stem, "pdf_probe_failed", "PyMuPDF could not inspect the local PDF.")
+        adapter = PdfDocumentAdapter("pdf", request.primary_backend)
+        result = adapter.extract_text(
+            request.item,
+            backend=request.primary_backend,
+            backend_chain=(request.primary_backend,),
+            allow_fallback=False,
+        )
+        if probes is not None:
+            result = replace(result, metadata={**result.metadata, "pages": [{**p, "status": "verified", "engine": self.name, "model": None, "digest": None, "fallback_used": False, "reason": None, "ovis_seconds": 0.0, "fallback_seconds": 0.0, "peak_vram_mb": None, "han_ratio": None, "slice_parameters": {}, "breakers": []} for p in probes]})
+        if self.name == "pdf.markitdown" and path and path.is_file() and path.suffix.lower() in {".docx", ".pptx", ".xlsx"} and result.error_code not in {"document_invalid_container", "document_extraction_failed"}:
+            from openmd.ocr.office import recover_office
+
+            item = replace(request.item, raw_meta={**request.item.raw_meta, "local_path": str(path)})
+            result = recover_office(item, result, allow_ocr=self.options.enable_network_backends, allow_external_resources=self.options.enable_network_backends)
+        return result
+
+
+class PDFTextLayerProvider:
+    name = "pdf.textlayer"
+
+    def descriptor(self):
+        return BackendDescriptor(name=self.name, layer="local", provider="openmd.pdf", description="PyMuPDF native page text; incomplete PDFs continue to OCR.", capabilities=(BackendCapability(platforms=("pdf",), source_types=("document",), input_types=("single", "batch", "collection", "generic"), source_levels=("native_text",), requires_network=False),))
+
+    def health(self):
+        import importlib.util
+
+        available = importlib.util.find_spec("pymupdf") is not None
+        return BackendHealth(ok=available, error_code=None if available else "pymupdf_missing")
+
+    def extract(self, request):
+        from openmd.ocr.documents import extract_pdf
+
+        return extract_pdf(request.item, backend=self.name)
+
+
+def register_backend_providers(
+    registry: BackendProviderRegistry,
+    options: BackendRuntimeOptions,
+) -> None:
+    registry.register("pdf.mock", PDFParserProvider(MockPDFParser()))
+    registry.register("pdf.markitdown", PDFParserProvider(MarkItDownPDFParser(), options))
+    registry.register("pdf.textlayer", PDFTextLayerProvider())
+    from openmd.backends.ocr import register_ocr_providers
+
+    register_ocr_providers(registry, options)

openmd/backends/registry.py

@@ -0,0 +1,234 @@
+"""Runtime backend provider registration helpers.
+
+This module centralizes core provider wiring so platform lanes can add
+platform-specific modules without repeatedly editing the CLI or central adapter
+registry. Real network/browser providers still need explicit CLI opt-in.
+"""
+
+from __future__ import annotations
+
+import importlib
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
+
+from .executor import (
+    BackendProviderRegistry,
+    DiscoveryProviderRegistry,
+    MockDiscoveryProvider,
+    create_safe_backend_provider_registry,
+    create_safe_discovery_provider_registry,
+)
+from .youtube_discovery import (
+    OpenCliYouTubeChannelProvider,
+    OpenCliYouTubePlaylistProvider,
+    YouTubeDataApiChannelProvider,
+    YouTubeDataApiPlaylistProvider,
+    YOUTUBE_DATA_API_DISCOVERY_BACKEND,
+    YOUTUBE_DATA_API_PLAYLIST_DISCOVERY_BACKEND,
+    YOUTUBE_DISCOVERY_BACKEND,
+    YOUTUBE_PLAYLIST_DISCOVERY_BACKEND,
+)
+from .youtube_transcript import YOUTUBE_TRANSCRIPT_BACKEND, YouTubeTranscriptApiProvider
+
+
+@dataclass(frozen=True)
+class BackendRuntimeOptions:
+    """Safe runtime gates shared by extraction and discovery provider setup."""
+
+    backend_override: str | None = None
+    requested_discovery_provider: str | None = None
+    enable_network_backends: bool = False
+    enable_browser_backends: bool = False
+    enable_media_download_backends: bool = False
+    enable_asr_backends: bool = False
+    enable_cloud_backends: bool = False
+    keep_media_artifacts: bool = False
+    media_artifact_dir: str | None = None
+
+
+ExtensionModuleLoader = Callable[[str], Iterable[object]]
+
+
+DISCOVERY_BACKEND_ALIASES: dict[str, str] = {
+    "api": YOUTUBE_DATA_API_DISCOVERY_BACKEND,
+    "data-api": YOUTUBE_DATA_API_DISCOVERY_BACKEND,
+    "data-api-playlist": YOUTUBE_DATA_API_PLAYLIST_DISCOVERY_BACKEND,
+    "mock": "mock.discovery",
+    "skeleton": "skeleton.discovery",
+    "opencli": YOUTUBE_DISCOVERY_BACKEND,
+    "opencli-playlist": YOUTUBE_PLAYLIST_DISCOVERY_BACKEND,
+    "playlist": YOUTUBE_PLAYLIST_DISCOVERY_BACKEND,
+    "youtube-data-api": YOUTUBE_DATA_API_DISCOVERY_BACKEND,
+    "youtube-data-api-playlist": YOUTUBE_DATA_API_PLAYLIST_DISCOVERY_BACKEND,
+    "youtube-playlist": YOUTUBE_PLAYLIST_DISCOVERY_BACKEND,
+    "wechat-article-exporter-api": "wechat-article-exporter.http",
+    "wechat-article-exporter-http": "wechat-article-exporter.http",
+    "weixin-exporter-api": "wechat-article-exporter.http",
+    "weixin-exporter-http": "wechat-article-exporter.http",
+    "weixin-http": "wechat-article-exporter.http",
+    "openmd-douyin": "opencli.openmd-douyin.creator-videos",
+    "openmd-douyin-creator-videos": "opencli.openmd-douyin.creator-videos",
+}
+
+
+def normalize_discovery_backend(value: str | None) -> str | None:
+    if value is None:
+        return None
+    return DISCOVERY_BACKEND_ALIASES.get(value, value)
+
+
+def create_runtime_backend_provider_registry(
+    platform: str,
+    options: BackendRuntimeOptions,
+    *,
+    extension_module_loader: ExtensionModuleLoader | None = None,
+) -> BackendProviderRegistry:
+    registry = create_safe_backend_provider_registry()
+    _register_builtin_backend_providers(registry, platform, options)
+    _register_extension_backend_providers(
+        registry,
+        platform,
+        options,
+        extension_module_loader=extension_module_loader,
+    )
+    from openmd.routing.providers import register_card_providers
+
+    register_card_providers(registry, platform, options)
+    return registry
+
+
+def create_runtime_discovery_provider_registry(
+    platform: str,
+    options: BackendRuntimeOptions,
+    *,
+    extension_module_loader: ExtensionModuleLoader | None = None,
+) -> DiscoveryProviderRegistry:
+    registry = create_safe_discovery_provider_registry()
+    requested_provider = normalize_discovery_backend(options.requested_discovery_provider)
+    if requested_provider is None and options.backend_override == "mock":
+        registry.register(
+            "mock.discovery",
+            MockDiscoveryProvider(),
+            platforms=(platform,),
+        )
+
+    should_register_network_provider = (
+        options.enable_network_backends
+        and platform == "youtube"
+        and (requested_provider is None or requested_provider == YOUTUBE_DATA_API_DISCOVERY_BACKEND)
+    )
+    if should_register_network_provider:
+        registry.register(
+            YOUTUBE_DATA_API_DISCOVERY_BACKEND,
+            YouTubeDataApiChannelProvider(),
+            platforms=("youtube",),
+            default_for_platforms=requested_provider is None,
+        )
+
+    should_register_network_playlist_provider = (
+        options.enable_network_backends
+        and platform == "youtube"
+        and (requested_provider is None or requested_provider == YOUTUBE_DATA_API_PLAYLIST_DISCOVERY_BACKEND)
+    )
+    if should_register_network_playlist_provider:
+        registry.register(
+            YOUTUBE_DATA_API_PLAYLIST_DISCOVERY_BACKEND,
+            YouTubeDataApiPlaylistProvider(),
+            platforms=("youtube",),
+            default_for_platforms=requested_provider == YOUTUBE_DATA_API_PLAYLIST_DISCOVERY_BACKEND,
+        )
+
+    should_register_browser_channel_provider = (
+        options.enable_browser_backends
+        and platform == "youtube"
+        and (requested_provider is None or requested_provider == YOUTUBE_DISCOVERY_BACKEND)
+    )
+    if should_register_browser_channel_provider:
+        registry.register(
+            YOUTUBE_DISCOVERY_BACKEND,
+            OpenCliYouTubeChannelProvider(),
+            platforms=("youtube",),
+            default_for_platforms=requested_provider is None and not should_register_network_provider,
+        )
+
+    should_register_browser_playlist_provider = (
+        options.enable_browser_backends
+        and platform == "youtube"
+        and (requested_provider is None or requested_provider == YOUTUBE_PLAYLIST_DISCOVERY_BACKEND)
+    )
+    if should_register_browser_playlist_provider:
+        registry.register(
+            YOUTUBE_PLAYLIST_DISCOVERY_BACKEND,
+            OpenCliYouTubePlaylistProvider(),
+            platforms=("youtube",),
+            default_for_platforms=requested_provider == YOUTUBE_PLAYLIST_DISCOVERY_BACKEND,
+        )
+
+    _register_extension_discovery_providers(
+        registry,
+        platform,
+        options,
+        extension_module_loader=extension_module_loader,
+    )
+    return registry
+
+
+def _register_builtin_backend_providers(
+    registry: BackendProviderRegistry,
+    platform: str,
+    options: BackendRuntimeOptions,
+) -> None:
+    if options.enable_network_backends and platform == "youtube":
+        registry.register(YOUTUBE_TRANSCRIPT_BACKEND, YouTubeTranscriptApiProvider())
+
+
+def _register_extension_backend_providers(
+    registry: BackendProviderRegistry,
+    platform: str,
+    options: BackendRuntimeOptions,
+    *,
+    extension_module_loader: ExtensionModuleLoader | None = None,
+) -> None:
+    for module in _load_backend_extension_modules(platform, extension_module_loader):
+        registrar = getattr(module, "register_backend_providers", None)
+        if registrar is not None:
+            if not callable(registrar):
+                raise TypeError("register_backend_providers must be callable")
+            registrar(registry, options)
+
+
+def _register_extension_discovery_providers(
+    registry: DiscoveryProviderRegistry,
+    platform: str,
+    options: BackendRuntimeOptions,
+    *,
+    extension_module_loader: ExtensionModuleLoader | None = None,
+) -> None:
+    for module in _load_backend_extension_modules(platform, extension_module_loader):
+        registrar = getattr(module, "register_discovery_providers", None)
+        if registrar is not None:
+            if not callable(registrar):
+                raise TypeError("register_discovery_providers must be callable")
+            registrar(registry, options)
+
+
+def _load_backend_extension_modules(
+    platform: str,
+    extension_module_loader: ExtensionModuleLoader | None = None,
+) -> tuple[object, ...]:
+    if extension_module_loader is not None:
+        return tuple(extension_module_loader(platform))
+
+    modules: list[object] = []
+    for module_name in _backend_extension_module_names(platform):
+        try:
+            modules.append(importlib.import_module(module_name))
+        except ModuleNotFoundError as exc:
+            if exc.name == module_name:
+                continue
+            raise
+    return tuple(modules)
+
+
+def _backend_extension_module_names(platform: str) -> tuple[str, str]:
+    return (f"openmd.backends.{platform}", f"openmd.backends.{platform}_providers")