openmd-cli 0.1.0__py3-none-any.whl

openmd/routing/providers.py

@@ -0,0 +1,158 @@
+"""Bridge declarative command cards into the existing backend executor."""
+
+from __future__ import annotations
+
+import hashlib
+import importlib.metadata
+import json
+import shutil
+import subprocess
+import tempfile
+from dataclasses import replace
+from pathlib import Path
+from typing import Any
+
+from openmd.backends.base import BackendCapability, BackendDescriptor, BackendHealth
+from openmd.core.local_config import load_local_config
+from openmd.core.models import TranscriptResult
+from openmd.core.secrets import sanitize_error_summary
+
+from .loader import load_catalog, route_trace
+from .readers import read_media, read_text
+from .runner import ToolError, isolated_environment, resolve_launcher, run_tool
+
+from openmd.runtime.tempdirs import owned_temporary_directory
+
+GATE_OPTIONS = {"network": "enable_network_backends", "browser": "enable_browser_backends", "media_download": "enable_media_download_backends", "asr": "enable_asr_backends", "cloud": "enable_cloud_backends"}
+
+
+def gates_open(capability: dict[str, Any], options) -> bool:
+    return all(gate not in {"account", "local_export"} and (gate == "local_compute" or bool(getattr(options, GATE_OPTIONS.get(gate, ""), False))) for gate in capability.get("gates", []))
+
+
+def installed_identity(card: dict[str, Any]) -> dict[str, Any]:
+    launcher = resolve_launcher(card)
+    digest = hashlib.sha256(launcher.entry_path.read_bytes()).hexdigest()
+    actual = None
+    if card["kind"] == "command":
+        for parent in [launcher.entry_path.parent, *list(launcher.entry_path.parents)[:3]]:
+            metadata = parent / "package.json"
+            if metadata.is_file():
+                try:
+                    actual = json.loads(metadata.read_text(encoding="utf-8")).get("version")
+                except (OSError, ValueError):
+                    pass
+                if actual:
+                    break
+    elif card["kind"] == "python_entry":
+        paths = load_local_config().get("tools", {}).get(card["id"], {}).get("paths")
+        wanted = card["id"].lower().replace("_", "-")
+        for dist in importlib.metadata.distributions(path=paths):
+            if str(dist.metadata.get("Name", "")).lower().replace("_", "-") == wanted:
+                actual = dist.version
+                break
+    if card["kind"] == "python_script":
+        actual = "sha256:" + digest
+        verified = card.get("installation", {}).get("sha256") == digest
+    else:
+        verified = actual == card.get("version")
+    locked_digest = card.get("installation", {}).get("sha256")
+    if locked_digest and locked_digest != digest:
+        verified = False
+    return {"tool_version": actual, "tool_version_verified": verified, "version_origin": "installed_metadata" if card["kind"] != "python_script" else "source_sha256", "entry_sha256": digest, "origin": launcher.origin}
+
+
+class CommandToolProvider:
+    def __init__(self, card: dict[str, Any], capability: dict[str, Any], options) -> None:
+        self.card, self.capability, self.options = card, capability, options
+        self.name = capability["id"]
+
+    def descriptor(self) -> BackendDescriptor:
+        gates = self.capability.get("gates", [])
+        return BackendDescriptor(name=self.name, layer="command_card", capabilities=(BackendCapability(platforms=tuple(self.capability["sources"]), source_types=("article", "audio", "video"), input_types=("single", "batch", "collection"), source_levels=(self.capability["source_level"],), requires_network="network" in gates, requires_credentials=bool(self.capability.get("requires_login"))),), description="Declared tool with validated artifact output.", provider=self.card["id"])
+
+    def health(self) -> BackendHealth:
+        if not gates_open(self.capability, self.options):
+            return BackendHealth(ok=False, error_code="tool_permission_required", error_summary="Explicit runtime gates are required for this tool.")
+        try:
+            identity = installed_identity(self.card)
+            if not identity["tool_version_verified"]:
+                return BackendHealth(ok=False, error_code="tool_version_mismatch", error_summary="Installed tool identity differs from its locked card; inspect doctor before updating the card.")
+            return BackendHealth(ok=True, dependencies={self.card["id"]: identity["tool_version"]})
+        except (ToolError, OSError, ValueError) as exc:
+            return BackendHealth(ok=False, error_code=getattr(exc, "code", "tool_not_installed"), error_summary=sanitize_error_summary(exc))
+
+    def extract(self, request) -> TranscriptResult:
+        if not gates_open(self.capability, self.options):
+            return self._failed("tool_permission_required", "Required runtime gates have not been enabled.")
+        if request.item.platform not in self.capability["sources"]:
+            return self._failed("tool_input_unsupported", "Tool card does not cover this source.")
+        metadata: dict[str, Any] = {}
+        try:
+            identity = installed_identity(self.card)
+            if not identity["tool_version_verified"]:
+                return self._failed("tool_version_mismatch", "Installed tool differs from its locked card; run openmd doctor --profile tools.")
+            with owned_temporary_directory(prefix="openmd-tool-", ignore_cleanup_errors=True) as tmp:
+                workdir = Path(tmp)
+                run = run_tool(self.card, self.capability, request.item, workdir)
+                metadata = {"tool_run": {"seconds": run.seconds, "exit_code": run.exit_code, "files": run.files, "card_id": self.card["id"], **identity}, **route_trace(request.item.platform, self.name)}
+                if self.card["id"] == "yutto":
+                    metadata["tool_run"]["anonymous_notice"] = "未提供登录认证信息" in run.stdout + run.stderr
+                if run.exit_code != 0:
+                    return replace(self._failed("tool_backend_exit", f"Tool exited with code {run.exit_code}; raw logs were not retained."), metadata=metadata)
+                if self.capability.get("then"):
+                    from openmd.runtime.asr import get_asr_backend
+
+                    media = read_media(workdir)
+                    result = get_asr_backend(self.capability["then"]).transcribe(media)
+                    metadata.update({"media_tool": self.card["id"], "media_bytes": media.stat().st_size, **result.metadata})
+                    result = replace(result, backend=self.name, source_level="media_asr" if result.has_usable_text else "partial_extraction", title=request.item.title or media.stem, source_notes=[*result.source_notes, f"Media fetched by {self.card['id']} and transcribed locally; command logs are not transcript evidence."])
+                    if getattr(self.options, "keep_media_artifacts", False) and getattr(self.options, "media_artifact_dir", None):
+                        destination = Path(self.options.media_artifact_dir)
+                        try:
+                            destination.mkdir(parents=True, exist_ok=True)
+                            with tempfile.NamedTemporaryFile(prefix=hashlib.sha256(request.item.url.encode()).hexdigest()[:12] + "-", suffix=media.suffix, dir=destination, delete=False) as dst, media.open("rb") as src:
+                                shutil.copyfileobj(src, dst)
+                            metadata["media_retained"] = True
+                        except OSError:
+                            metadata["media_retained"] = False
+                            result = replace(result, source_notes=[*result.source_notes, "Transcript succeeded, but the optional media copy could not be retained."])
+                else:
+                    result = read_text(self.capability, run, workdir)
+                return replace(result, metadata={**result.metadata, **metadata})
+        except (ToolError, OSError, ValueError) as exc:
+            return replace(self._failed(getattr(exc, "code", "tool_artifact_invalid"), sanitize_error_summary(exc) or "Tool failed."), metadata=metadata)
+
+    def _failed(self, code: str, detail: str) -> TranscriptResult:
+        return TranscriptResult(text="", status="failed", source_level="partial_extraction", backend=self.name, source_notes=["The declared tool did not produce validated source content."], error_code=code, error_summary=detail)
+
+
+def register_card_providers(registry, platform: str, options) -> None:
+    catalog = load_catalog()
+    requested = options.backend_override
+    for capability in catalog.capabilities.values():
+        card = catalog.cards[capability["card_id"]]
+        if card["kind"] != "builtin" and platform in capability["sources"] and (requested is None or requested == capability["id"]) and gates_open(capability, options):
+            registry.register(capability["id"], CommandToolProvider(card, capability, options))
+
+
+def doctor_tools(*, probe_commands: bool = False) -> dict[str, Any]:
+    reports: list[dict[str, Any]] = []
+    for card in load_catalog().cards.values():
+        if card["kind"] == "builtin":
+            continue
+        record = {"tool": card["id"], "kind": card["kind"], "declared_version": card["version"], "requires_login": any(c.get("requires_login", False) for c in card["capabilities"]), "gates": sorted({g for c in card["capabilities"] for g in c.get("gates", [])}), "resolved": False, "origin": "none", "version": None, "version_probe": "not_requested"}
+        try:
+            identity = installed_identity(card)
+            record.update(identity)
+            record.update({"resolved": True, "version": identity["tool_version"]})
+            if not identity["tool_version_verified"]:
+                record["error_code"] = "tool_version_mismatch"
+            # Local metadata/hash inspection is sufficient for these cards; never
+            # run a tool's ordinary operation just to discover a version.
+            if probe_commands:
+                record["version_probe"] = "installed_metadata_and_source_hash"
+        except (ToolError, OSError, ValueError) as exc:
+            record["error_code"] = getattr(exc, "code", "tool_not_installed")
+        reports.append(record)
+    return {"ok": all(r["resolved"] and r.get("tool_version_verified") for r in reports), "tools": reports, "network_access": "not_probed", "private_session_state": "not_read"}

openmd/routing/readers.py

@@ -0,0 +1,108 @@
+"""Read declared artifacts; command logs and metadata never count as source text."""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+from openmd.core.content_validation import validate_extraction_content
+from openmd.core.models import TranscriptResult
+
+from .runner import ToolError, ToolRun
+
+MEDIA_EXTENSIONS = frozenset({".m4a", ".mp3", ".aac", ".opus", ".wav", ".mp4", ".webm", ".ogg", ".flac"})
+
+
+def field_value(data: Any, path: str | list[str]) -> str | None:
+    if isinstance(path, list):
+        return next((value for key in path if (value := field_value(data, key))), None)
+    current = data
+    for key in path.split("."):
+        if not isinstance(current, dict) or key not in current:
+            return None
+        current = current[key]
+    return current if isinstance(current, str) and current.strip() else None
+
+
+def contained_file(workdir: Path, relative: str) -> Path:
+    path = (workdir / relative).resolve()
+    if not path.is_relative_to(workdir.resolve()) or not path.is_file():
+        raise ToolError("tool_artifact_missing")
+    return path
+
+
+def subtitle_text(text: str) -> str:
+    lines: list[str] = []
+    timestamp: str | None = None
+    for line in text.splitlines():
+        match = re.match(r"\s*((?:\d{2}:)?\d{2}:\d{2})[,.]\d+\s*-->", line)
+        if match:
+            timestamp = match.group(1)
+            if timestamp.count(":") == 1:
+                timestamp = "00:" + timestamp
+        elif not line.strip() or line.strip().isdigit() or line.strip() == "WEBVTT":
+            continue
+        elif timestamp:
+            value = re.sub(r"<[^>]+>", "", line).strip()
+            if value:
+                lines.append(f"[{timestamp}] {value}")
+    return "\n".join(lines)
+
+
+def read_media(workdir: Path) -> Path:
+    media = [p for p in workdir.rglob("*") if p.is_file() and p.suffix.lower() in MEDIA_EXTENSIONS and p.resolve().is_relative_to(workdir.resolve()) and p.stat().st_size > 0]
+    if not media:
+        raise ToolError("tool_media_missing", "The tool returned no media; metadata and command logs are insufficient.")
+    return max(media, key=lambda p: p.stat().st_size)
+
+
+def read_text(capability: dict[str, Any], run: ToolRun, workdir: Path) -> TranscriptResult:
+    reader = capability["reader"]
+    title = author = published_at = None
+    kind = reader["type"]
+    if kind == "markdown_file":
+        path = contained_file(workdir, reader["path"])
+        if path.stat().st_size > 16 * 1024 * 1024:
+            raise ToolError("tool_output_too_large")
+        text = path.read_text(encoding="utf-8-sig")
+        heading = re.search(r"^#\s+(.+)$", text, re.M)
+        title = heading.group(1).strip() if heading else None
+    elif kind in {"json_stdout", "json_file"}:
+        if kind == "json_file":
+            path = contained_file(workdir, reader["path"])
+            if path.stat().st_size > 16 * 1024 * 1024:
+                raise ToolError("tool_output_too_large")
+            raw_json = path.read_text(encoding="utf-8-sig")
+        else:
+            raw_json = run.stdout
+        try:
+            data = json.loads(raw_json)
+        except ValueError as exc:
+            raise ToolError("tool_json_invalid") from exc
+        if not isinstance(data, dict) or data.get("error") or data.get("success") is False:
+            raise ToolError("tool_upstream_failed", "Tool JSON contains an upstream error.")
+        fields = reader["fields"]
+        text = field_value(data, fields["text"]) or ""
+        title = field_value(data, fields.get("title", "title"))
+        author = field_value(data, fields.get("author", "author"))
+        published_at = field_value(data, fields.get("published_at", "published_at"))
+        if not title and text and reader.get("title_from_first_line", False):
+            title = text.splitlines()[0][:120]
+    elif kind == "subtitle_dir":
+        paths = sorted(p for p in workdir.rglob("*") if p.suffix.lower() in {".srt", ".vtt"} and p.is_file() and p.resolve().is_relative_to(workdir.resolve()))
+        if not paths:
+            raise ToolError("tool_subtitle_missing")
+        text = subtitle_text(paths[0].read_text(encoding="utf-8-sig"))
+    else:
+        raise ToolError("tool_reader_invalid")
+    text = text.strip()
+    if len(text) < capability.get("validation", {}).get("min_chars", 1):
+        raise ToolError("tool_content_empty", "Tool returned insufficient source text.")
+    if len(text) < 3000 and re.search(r"(?im)^\s*(?:#\s*)?(?:just a moment[.!…]*|verify you are human|请完成验证|请先登录|访问过于频繁|安全验证)\s*$", text):
+        raise ToolError("tool_challenge_page", "A challenge/login page is not source content.")
+    if re.match(r"\s*(?:<!doctype html|<html)", text, re.I):
+        raise ToolError("tool_html_instead_of_markdown")
+    result = validate_extraction_content(TranscriptResult(text=text + "\n", status="verified", source_level=capability["source_level"], backend=capability["id"], title=title, author=author, published_at=published_at, source_notes=["Original source text read from the tool's declared artifact.", *([capability["coverage_note"]] if capability.get("coverage_note") else [])]))
+    return result