openmd-cli 0.1.0__py3-none-any.whl

openmd/adapters/qq_group/parsers/__init__.py

@@ -0,0 +1 @@
+"""Parsers for user-provided QQ group export files."""

openmd/adapters/qq_group/parsers/html.py

@@ -0,0 +1,60 @@
+"""Small parser for user-provided QQ HTML group exports."""
+
+from __future__ import annotations
+
+import re
+from html import unescape
+from typing import Any
+
+
+_MESSAGE_RE = re.compile(
+    r"<(?P<tag>div|li)[^>]+class=[\"'][^\"']*(?:message|msg)[^\"']*[\"'][^>]*>(?P<body>.*?)</(?P=tag)>",
+    re.IGNORECASE | re.DOTALL,
+)
+
+
+def parse_qq_html_export(text: str) -> list[dict[str, Any]]:
+    messages: list[dict[str, Any]] = []
+    blocks = [match.group("body") for match in _MESSAGE_RE.finditer(text)]
+    if not blocks:
+        blocks = [line for line in text.splitlines() if "sender" in line or "content" in line]
+
+    for index, block in enumerate(blocks, start=1):
+        message = _parse_message_block(block, index=index)
+        if message is not None:
+            messages.append(message)
+    return messages
+
+
+def _parse_message_block(block: str, *, index: int) -> dict[str, Any] | None:
+    timestamp = _class_text(block, "time") or _class_text(block, "timestamp")
+    speaker = _class_text(block, "sender") or _class_text(block, "nickname") or _class_text(block, "name")
+    content = _class_text(block, "content") or _class_text(block, "text") or _strip_tags(block)
+    if not content:
+        return None
+    return {
+        "id": f"html_{index:04d}",
+        "timestamp": timestamp or "unknown-time",
+        "speaker": speaker or "unknown",
+        "text": content,
+        "attachments": [],
+        "links": [],
+        "source_map": [],
+    }
+
+
+def _class_text(block: str, class_name: str) -> str | None:
+    pattern = re.compile(
+        rf"<[^>]+class=[\"'][^\"']*{re.escape(class_name)}[^\"']*[\"'][^>]*>(.*?)</[^>]+>",
+        re.IGNORECASE | re.DOTALL,
+    )
+    match = pattern.search(block)
+    if not match:
+        return None
+    return _strip_tags(match.group(1))
+
+
+def _strip_tags(value: str) -> str:
+    text = re.sub(r"<[^>]+>", " ", value)
+    text = unescape(text)
+    return " ".join(text.split())

openmd/adapters/qq_group/parsers/json.py

@@ -0,0 +1,71 @@
+"""Parser for qq-chat-exporter-style JSON group exports."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable
+from typing import Any
+
+
+def parse_qq_json_export(text: str) -> list[dict[str, Any]]:
+    try:
+        payload = json.loads(text)
+    except json.JSONDecodeError as exc:
+        raise ValueError("QQ JSON export is not valid JSON") from exc
+
+    messages: list[dict[str, Any]] = []
+    for index, row in enumerate(_candidate_rows(payload), start=1):
+        if not isinstance(row, dict):
+            continue
+        message = _parse_message_row(row, index=index)
+        if message is not None:
+            messages.append(message)
+    return messages
+
+
+def _candidate_rows(payload: object) -> Iterable[object]:
+    if isinstance(payload, list):
+        return payload
+    if not isinstance(payload, dict):
+        return ()
+    for key in ("messages", "items", "list", "data", "records", "results"):
+        value = payload.get(key)
+        if isinstance(value, list):
+            return value
+        if isinstance(value, dict):
+            nested = list(_candidate_rows(value))
+            if nested:
+                return nested
+    return ()
+
+
+def _parse_message_row(row: dict[str, Any], *, index: int) -> dict[str, Any] | None:
+    text = _first_string(row, "text", "content", "message", "msg")
+    speaker = _first_string(row, "speaker", "sender", "from", "author", "nickname", "name")
+    timestamp = _first_string(row, "timestamp", "time", "datetime", "date", "created_at")
+    attachments = _list_or_empty(row.get("attachments") or row.get("files") or row.get("images"))
+    if not text and not attachments:
+        return None
+    return {
+        "id": _first_string(row, "id", "msg_id", "message_id") or f"json_{index:04d}",
+        "timestamp": timestamp or "unknown-time",
+        "speaker": speaker or "unknown",
+        "text": text or "",
+        "attachments": attachments,
+        "links": _list_or_empty(row.get("links")),
+        "source_map": [],
+    }
+
+
+def _first_string(row: dict[str, Any], *keys: str) -> str | None:
+    for key in keys:
+        value = row.get(key)
+        if isinstance(value, str) and value.strip():
+            return value.strip()
+        if isinstance(value, int):
+            return str(value)
+    return None
+
+
+def _list_or_empty(value: object) -> list[Any]:
+    return value if isinstance(value, list) else []

openmd/adapters/registry.py

@@ -0,0 +1,149 @@
+"""Adapter registry and default backend preferences."""
+
+from __future__ import annotations
+
+import importlib
+from dataclasses import dataclass
+
+from collections.abc import Callable, Iterable
+
+from openmd.core.detector import PLATFORM_IDS, detect_platform
+
+from .base import PlatformAdapter, StubPlatformAdapter
+from .opencli import OpenCliPlatformAdapter
+
+
+AdapterFactory = Callable[[str, str], PlatformAdapter]
+AdapterFactoryLoader = Callable[[str], AdapterFactory | None]
+
+
+OPENCLI_ADAPTER_PLATFORMS: frozenset[str] = frozenset(
+    {
+        "bilibili",
+        "youtube",
+        "weixin",
+        "xiaoyuzhou",
+        "xiaohongshu",
+        "xiaoe",
+        "x",
+    }
+)
+
+
+from openmd.routing.loader import default_backend_chains
+
+
+DEFAULT_BACKENDS: dict[str, tuple[str, ...]] = default_backend_chains(effective=False)
+
+@dataclass(frozen=True)
+class BackendSelection:
+    platform: str
+    primary: str
+    fallbacks: tuple[str, ...]
+
+    @property
+    def chain(self) -> tuple[str, ...]:
+        return (self.primary, *self.fallbacks)
+
+
+class AdapterRegistry:
+    def __init__(self) -> None:
+        self._adapters: dict[str, PlatformAdapter] = {}
+        self._backends: dict[str, tuple[str, ...]] = default_backend_chains()
+
+    def register(
+        self,
+        adapter: PlatformAdapter,
+        *,
+        backends: Iterable[str] | None = None,
+    ) -> None:
+        self._adapters[adapter.id] = adapter
+        if backends is not None:
+            self._backends[adapter.id] = tuple(backends)
+
+    def get_adapter(self, platform: str) -> PlatformAdapter:
+        try:
+            return self._adapters[platform]
+        except KeyError as exc:
+            raise KeyError(f"no adapter registered for platform: {platform}") from exc
+
+    def adapter_for(self, input_value: str) -> PlatformAdapter:
+        return self.get_adapter(detect_platform(input_value))
+
+    def backend_chain(
+        self,
+        platform: str,
+        requested: str | None = None,
+        *,
+        allow_fallback: bool = True,
+    ) -> tuple[str, ...]:
+        if requested:
+            return (requested,)
+        chain = self._backends.get(platform, ("mock",))
+        if allow_fallback:
+            return chain
+        return chain[:1]
+
+    def select_backend(
+        self,
+        platform: str,
+        requested: str | None = None,
+        *,
+        allow_fallback: bool = True,
+    ) -> BackendSelection:
+        chain = self.backend_chain(platform, requested, allow_fallback=allow_fallback)
+        primary = chain[0]
+        fallbacks = chain[1:] if allow_fallback else ()
+        return BackendSelection(platform=platform, primary=primary, fallbacks=fallbacks)
+
+
+def create_default_registry(
+    *,
+    adapter_factory_loader: AdapterFactoryLoader | None = None,
+) -> AdapterRegistry:
+    registry = AdapterRegistry()
+    load_factory = adapter_factory_loader or load_adapter_factory
+    for platform_id in PLATFORM_IDS:
+        backend = registry.backend_chain(platform_id)[0]
+        registry.register(_create_platform_adapter(platform_id, backend, load_factory))
+    return registry
+
+
+def _create_platform_adapter(
+    platform_id: str,
+    default_backend: str,
+    load_factory: AdapterFactoryLoader,
+) -> PlatformAdapter:
+    factory = load_factory(platform_id)
+    if factory is not None:
+        adapter = factory(platform_id, default_backend)
+        if adapter.id != platform_id:
+            raise ValueError(
+                f"adapter factory for {platform_id} returned adapter id {adapter.id!r}"
+            )
+        return adapter
+    if platform_id in OPENCLI_ADAPTER_PLATFORMS:
+        return OpenCliPlatformAdapter(platform_id, default_backend)
+    return StubPlatformAdapter(platform_id, default_backend)
+
+
+def load_adapter_factory(platform_id: str) -> AdapterFactory | None:
+    """Load an optional platform adapter factory by naming convention.
+
+    Platform lanes can add ``openmd.adapters.<platform_id>`` with a
+    ``create_adapter(platform_id, default_backend)`` function. The core registry
+    will pick it up without platform lanes editing this shared file.
+    """
+    module_name = f"openmd.adapters.{platform_id}"
+    try:
+        module = importlib.import_module(module_name)
+    except ModuleNotFoundError as exc:
+        if exc.name == module_name:
+            return None
+        raise
+    factory = getattr(module, "create_adapter", None)
+    if factory is None:
+        return None
+    if not callable(factory):
+        raise TypeError(f"{module_name}.create_adapter must be callable")
+    return factory

openmd/adapters/screenshot/__init__.py

@@ -0,0 +1,11 @@
+"""V1 long screenshot OCR adapter factory."""
+
+from __future__ import annotations
+
+from openmd.adapters.base import PlatformAdapter
+
+from .adapter import ScreenshotOcrAdapter
+
+
+def create_adapter(platform_id: str, default_backend: str) -> PlatformAdapter:
+    return ScreenshotOcrAdapter(platform_id, default_backend)

openmd/adapters/screenshot/adapter.py

@@ -0,0 +1,154 @@
+"""V1 adapter for long screenshot OCR package inputs.
+
+This lane is local and read-only by default. It resolves image files or folders
+into ``ContentItem`` records and leaves OCR/layout extraction to explicit
+backends. Raw screenshots are referenced as sources and are not copied into
+package assets.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from urllib.parse import urlparse
+import hashlib
+
+from openmd.adapters.base import PlatformAdapter
+from openmd.adapters.local_audio.adapter import local_path_from_input
+from openmd.core.detector import IMAGE_TEXT_EXTENSIONS, extract_first_url
+from openmd.core.models import ContentItem, TranscriptResult
+
+from .backends.rapidocr import RapidOCRBackend
+from .ocr import MissingOCRBackend, MockOCRBackend
+
+
+class ScreenshotOcrAdapter(PlatformAdapter):
+    """Resolve local screenshots, folders, and image URL references without side effects."""
+
+    def __init__(self, platform_id: str = "screenshot", default_backend: str = "ocr.rapidocr") -> None:
+        self.id = platform_id
+        self.default_backend = default_backend
+
+    def can_handle(self, input_value: str) -> bool:
+        return _is_image_text_input(input_value)
+
+    def resolve_items(
+        self,
+        input_value: str,
+        *,
+        limit: int | None = None,
+        all: bool = False,
+    ) -> list[ContentItem]:
+        path = local_path_from_input(input_value)
+        if path is not None and path.is_dir():
+            files = sorted(candidate for candidate in path.rglob("*") if _is_image_text_path(candidate))
+            if limit is not None and not all:
+                files = files[: max(0, limit)]
+            return [_content_item_for_path(candidate, platform=self.id) for candidate in files]
+
+        if path is not None:
+            return [_content_item_for_path(path, platform=self.id)]
+
+        target = extract_first_url(input_value)
+        return [_content_item_for_reference(target, platform=self.id)]
+
+    def extract_text(
+        self,
+        item: ContentItem,
+        *,
+        backend: str | None = None,
+        backend_chain: tuple[str, ...] | None = None,
+        allow_fallback: bool = True,
+    ) -> TranscriptResult:
+        selected = (backend_chain or (backend or self.default_backend,))[0]
+        source_ref = _item_source_ref(item)
+        title = item.title or _title_from_source_ref(source_ref)
+        if selected == "screenshot.mock":
+            return MockOCRBackend().recognize(source_ref, title=title)
+        if selected == "ocr.rapidocr":
+            return RapidOCRBackend().recognize(source_ref, title=title)
+        return MissingOCRBackend(selected).recognize(source_ref, title=title)
+
+
+def _content_item_for_path(path: Path, *, platform: str) -> ContentItem:
+    source_ref = path.resolve().as_uri() if path.exists() else str(path)
+    content_id = _image_content_id(path)
+    return ContentItem(
+        platform=platform,
+        input_type="single",
+        url=source_ref,
+        id=content_id,
+        title=path.stem,
+        source_type="image_text",
+        raw_meta={
+            "profile": "document",
+            "extraction_contract_version": "ocr-layout-v2",
+            "permission_scope": "local_file",
+            "local_path": str(path),
+            "content_id": content_id,
+        },
+    )
+
+
+def _content_item_for_reference(source_ref: str, *, platform: str) -> ContentItem:
+    return ContentItem(
+        platform=platform,
+        input_type="single",
+        url=source_ref,
+        id=_stable_id(source_ref),
+        title=_title_from_source_ref(source_ref),
+        source_type="image_text",
+        raw_meta={
+            "profile": "document",
+            "extraction_contract_version": "ocr-layout-v2",
+            "permission_scope": "user_accessible",
+            "content_id": _stable_id(source_ref),
+        },
+    )
+
+
+def _is_image_text_input(input_value: str) -> bool:
+    path = local_path_from_input(input_value)
+    if path is not None:
+        if path.is_dir():
+            return _directory_contains_image_text(path)
+        return _is_image_text_path(path)
+    parsed = urlparse(extract_first_url(input_value))
+    return parsed.path.lower().endswith(IMAGE_TEXT_EXTENSIONS)
+
+
+def _is_image_text_path(path: Path) -> bool:
+    return path.suffix.lower() in IMAGE_TEXT_EXTENSIONS
+
+
+def _directory_contains_image_text(path: Path) -> bool:
+    try:
+        return any(candidate.is_file() and _is_image_text_path(candidate) for candidate in path.rglob("*"))
+    except OSError:
+        return False
+
+
+def _image_content_id(path: Path) -> str:
+    if path.is_file():
+        digest = hashlib.sha256()
+        with path.open("rb") as handle:
+            for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+                digest.update(chunk)
+        return digest.hexdigest()[:12]
+    return _stable_id(str(path))
+
+
+def _item_source_ref(item: ContentItem) -> str:
+    local_path = item.raw_meta.get("local_path") if isinstance(item.raw_meta, dict) else None
+    if isinstance(local_path, str) and local_path.strip():
+        return local_path
+    return item.url
+
+
+def _title_from_source_ref(source_ref: str) -> str:
+    parsed = urlparse(source_ref)
+    path = Path(parsed.path if parsed.scheme else source_ref)
+    return path.stem or "screenshot"
+
+
+def _stable_id(value: str) -> str:
+    return hashlib.sha256(value.strip().encode("utf-8")).hexdigest()[:12]

openmd/adapters/screenshot/backends/__init__.py

@@ -0,0 +1 @@
+"""Optional screenshot OCR backend placeholders."""

openmd/adapters/screenshot/backends/paddle.py

@@ -0,0 +1,15 @@
+"""PaddleOCR screenshot backend placeholder.
+
+The real OCR engine is intentionally not imported or executed by the
+mock/gated lane. Future work can replace this placeholder after dependency,
+privacy, and quality smoke are explicitly approved.
+"""
+
+from __future__ import annotations
+
+from openmd.adapters.screenshot.ocr import MissingOCRBackend
+
+
+class PaddleOCRBackend(MissingOCRBackend):
+    def __init__(self) -> None:
+        super().__init__("ocr.paddle")

openmd/adapters/screenshot/backends/rapidocr.py

@@ -0,0 +1,176 @@
+"""Optional RapidOCR backend for local screenshots/images.
+
+RapidOCR is imported lazily. The backend only reads local files or ``file://``
+references and never captures screens, opens accounts, or fetches remote image
+URLs.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import math
+from pathlib import Path
+from typing import Any
+
+from openmd.adapters.local_audio.adapter import local_path_from_input
+from openmd.adapters.screenshot.ocr import MissingOCRBackend, OCRBackend, classify_ocr_profile
+from openmd.core.models import TranscriptResult
+from openmd.core.secrets import sanitize_error_summary
+
+
+class RapidOCRBackend(OCRBackend):
+    name = "ocr.rapidocr"
+
+    def is_available(self) -> bool:
+        return importlib.util.find_spec("rapidocr_onnxruntime") is not None
+
+    def recognize(self, source_ref: str, *, title: str) -> TranscriptResult:
+        source_path = _local_existing_path(source_ref)
+        if source_path is None:
+            return _failed_result(
+                self.name,
+                source_ref,
+                title,
+                "ocr_local_file_required",
+                "RapidOCR requires a local image file path or file:// reference.",
+            )
+
+        if not self.is_available():
+            return MissingOCRBackend(self.name).recognize(str(source_path), title=title)
+
+        try:
+            from rapidocr_onnxruntime import RapidOCR
+
+            rows, _elapsed = RapidOCR()(str(source_path))
+        except Exception as exc:  # pragma: no cover - dependency-specific safety net
+            return _failed_result(self.name, str(source_path), title, "ocr_extraction_failed", exc)
+
+        lines, confidences = _extract_lines(rows)
+        text = "\n".join(lines).strip()
+        if not text:
+            return _failed_result(
+                self.name,
+                str(source_path),
+                title,
+                "ocr_empty_text",
+                "RapidOCR returned no recognized text.",
+            )
+
+        document_title = title or source_path.stem or "screenshot"
+        body = _with_title(document_title, text)
+        source_map, ambiguous = _layout_evidence(rows)
+        return TranscriptResult(
+            text=body,
+            status="partial" if ambiguous else "verified",
+            source_level="ocr",
+            backend=self.name,
+            source_notes=[
+                "RapidOCR recognized text from a local screenshot/image.",
+                "Raw screenshot file was referenced, not copied into the package.",
+                *(["Side-by-side OCR blocks have ambiguous reading order; verify against the source image."] if ambiguous else []),
+            ],
+            title=document_title,
+            metadata={
+                "profile": classify_ocr_profile(body),
+                "ocr_backend": self.name,
+                "source_ref": str(source_path),
+                "block_count": len(lines),
+                "average_confidence": _average(confidences),
+                "source_map": source_map,
+                "reading_order": "ambiguous" if ambiguous else "engine_order_not_independently_verified",
+            },
+        )
+
+
+def _local_existing_path(source_ref: str) -> Path | None:
+    path = local_path_from_input(source_ref)
+    if path is None:
+        return None
+    return path if path.is_file() else None
+
+
+def _extract_lines(rows: Any) -> tuple[list[str], list[float]]:
+    if not rows:
+        return [], []
+    lines: list[str] = []
+    confidences: list[float] = []
+    for row in rows:
+        if not isinstance(row, (list, tuple)) or len(row) < 2:
+            continue
+        text = row[1]
+        if isinstance(text, str) and text.strip():
+            lines.append(text.strip())
+        if len(row) >= 3 and isinstance(row[2], (int, float)):
+            confidences.append(float(row[2]))
+    return lines, confidences
+
+
+def _average(values: list[float]) -> float | None:
+    if not values:
+        return None
+    return round(sum(values) / len(values), 4)
+
+
+def _layout_evidence(rows: Any) -> tuple[list[dict[str, Any]], bool]:
+    """Retain pixel locators and flag side-by-side blocks without inventing order."""
+    boxes: list[dict[str, Any]] = []
+    if not isinstance(rows, (list, tuple)):
+        return boxes, False
+    line_number = 0
+    for row in rows:
+        if not isinstance(row, (list, tuple)) or len(row) < 2 or not isinstance(row[1], str) or not row[1].strip():
+            continue
+        line_number += 1
+        try:
+            points = [(float(point[0]), float(point[1])) for point in row[0]]
+            if len(points) < 4 or not all(math.isfinite(v) for point in points for v in point):
+                continue
+            xs, ys = zip(*points)
+            bbox = [min(xs), min(ys), max(xs), max(ys)]
+            boxes.append({"line": line_number, "bbox": bbox, "unit": "pixel"})
+        except (TypeError, ValueError, IndexError):
+            continue
+    # Only compare vertically overlapping blocks; this also flags table-like
+    # layouts, whose semantics cannot be certified by text confidence alone.
+    active: list[list[float]] = []
+    ambiguous = False
+    for entry in sorted(boxes, key=lambda entry: entry["bbox"][1]):
+        box = entry["bbox"]
+        active = [other for other in active if other[3] > box[1]]
+        for other in active:
+            overlap = min(other[3], box[3]) - max(other[1], box[1])
+            height = min(other[3] - other[1], box[3] - box[1])
+            gap = max(box[0] - other[2], other[0] - box[2])
+            if height > 0 and overlap >= height / 2 and gap > height:
+                ambiguous = True
+        active.append(box)
+    return boxes, ambiguous
+
+
+def _with_title(title: str, text: str) -> str:
+    if text.lstrip().startswith("#"):
+        return text if text.endswith("\n") else f"{text}\n"
+    return f"# {title}\n\n{text}\n"
+
+
+def _failed_result(
+    backend: str,
+    source_ref: str,
+    title: str,
+    code: str,
+    error: object,
+) -> TranscriptResult:
+    return TranscriptResult(
+        text="",
+        status="failed",
+        source_level="partial_extraction",
+        backend=backend,
+        source_notes=[
+            "OCR backend did not produce usable text.",
+            "Raw screenshot file was referenced, not copied into the package.",
+        ],
+        error_code=code,
+        error_summary=sanitize_error_summary(error),
+        title=title or "screenshot",
+        metadata={"profile": "document", "ocr_backend": backend, "source_ref": source_ref},
+    )

openmd/adapters/screenshot/backends/tesseract.py

@@ -0,0 +1,15 @@
+"""Tesseract screenshot backend placeholder.
+
+The real OCR engine is intentionally not imported or executed by the
+mock/gated lane. Future work can replace this placeholder after dependency,
+privacy, and quality smoke are explicitly approved.
+"""
+
+from __future__ import annotations
+
+from openmd.adapters.screenshot.ocr import MissingOCRBackend
+
+
+class TesseractOCRBackend(MissingOCRBackend):
+    def __init__(self) -> None:
+        super().__init__("ocr.tesseract")