openmd-cli 0.1.0__py3-none-any.whl

openmd/adapters/pdf/__init__.py

@@ -0,0 +1,11 @@
+"""V1 PDF document adapter factory."""
+
+from __future__ import annotations
+
+from openmd.adapters.base import PlatformAdapter
+
+from .adapter import PdfDocumentAdapter
+
+
+def create_adapter(platform_id: str, default_backend: str) -> PlatformAdapter:
+    return PdfDocumentAdapter(platform_id, default_backend)

openmd/adapters/pdf/adapter.py

@@ -0,0 +1,153 @@
+"""V1 adapter for local PDF and Office document inputs.
+
+This lane is local and read-only by default.  It resolves PDF files or folders
+into ``ContentItem`` records and leaves parsing to explicit parser backends.
+Raw documents are referenced as sources and are not copied into package assets.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from urllib.parse import urlparse
+import hashlib
+
+from openmd.adapters.base import PlatformAdapter
+from openmd.adapters.local_audio.adapter import local_path_from_input
+from openmd.core.detector import DOCUMENT_EXTENSIONS, extract_first_url
+from openmd.core.models import ContentItem, TranscriptResult
+
+from .backends.markitdown import MarkItDownPDFParser
+from .parser import MissingPDFParser, MockPDFParser
+
+
+class PdfDocumentAdapter(PlatformAdapter):
+    """Resolve local PDF files, folders, and PDF URL references without side effects."""
+
+    def __init__(self, platform_id: str = "pdf", default_backend: str = "pdf.markitdown") -> None:
+        self.id = platform_id
+        self.default_backend = default_backend
+
+    def can_handle(self, input_value: str) -> bool:
+        return _is_pdf_input(input_value)
+
+    def resolve_items(
+        self,
+        input_value: str,
+        *,
+        limit: int | None = None,
+        all: bool = False,
+    ) -> list[ContentItem]:
+        path = local_path_from_input(input_value)
+        if path is not None and path.is_dir():
+            files = sorted(candidate for candidate in path.rglob("*") if candidate.is_file() and _is_pdf_path(candidate))
+            if limit is not None and not all:
+                files = files[: max(0, limit)]
+            return [_content_item_for_path(candidate, platform=self.id) for candidate in files]
+
+        if path is not None:
+            return [_content_item_for_path(path, platform=self.id)]
+
+        target = extract_first_url(input_value)
+        return [_content_item_for_reference(target, platform=self.id)]
+
+    def extract_text(
+        self,
+        item: ContentItem,
+        *,
+        backend: str | None = None,
+        backend_chain: tuple[str, ...] | None = None,
+        allow_fallback: bool = True,
+    ) -> TranscriptResult:
+        selected = (backend_chain or (backend or self.default_backend,))[0]
+        source_ref = _item_source_ref(item)
+        title = item.title or _title_from_source_ref(source_ref)
+        if selected == "pdf.mock":
+            return MockPDFParser().parse(source_ref, title=title)
+        if selected == "pdf.markitdown":
+            return MarkItDownPDFParser().parse(source_ref, title=title)
+        return MissingPDFParser(selected).parse(source_ref, title=title)
+
+
+def _content_item_for_path(path: Path, *, platform: str) -> ContentItem:
+    source_ref = path.resolve().as_uri() if path.exists() else str(path)
+    content_id = _pdf_content_id(path)
+    return ContentItem(
+        platform=platform,
+        input_type="single",
+        url=source_ref,
+        id=content_id,
+        title=path.stem,
+        source_type="document",
+        raw_meta={
+            "profile": "document",
+            "extraction_contract_version": "document-native-v2",
+            "permission_scope": "local_file",
+            "local_path": str(path),
+            "content_id": content_id,
+        },
+    )
+
+
+def _content_item_for_reference(source_ref: str, *, platform: str) -> ContentItem:
+    return ContentItem(
+        platform=platform,
+        input_type="single",
+        url=source_ref,
+        id=_stable_id(source_ref),
+        title=_title_from_source_ref(source_ref),
+        source_type="document",
+        raw_meta={
+            "profile": "document",
+            "extraction_contract_version": "document-native-v2",
+            "permission_scope": "user_accessible",
+            "content_id": _stable_id(source_ref),
+        },
+    )
+
+
+def _is_pdf_input(input_value: str) -> bool:
+    path = local_path_from_input(input_value)
+    if path is not None:
+        if path.is_dir():
+            return _directory_contains_pdf(path)
+        return _is_pdf_path(path)
+    parsed = urlparse(extract_first_url(input_value))
+    return parsed.path.lower().endswith(DOCUMENT_EXTENSIONS)
+
+
+def _is_pdf_path(path: Path) -> bool:
+    return path.suffix.lower() in DOCUMENT_EXTENSIONS
+
+
+def _directory_contains_pdf(path: Path) -> bool:
+    try:
+        return any(candidate.is_file() and _is_pdf_path(candidate) for candidate in path.rglob("*"))
+    except OSError:
+        return False
+
+
+def _pdf_content_id(path: Path) -> str:
+    if path.is_file():
+        digest = hashlib.sha256()
+        with path.open("rb") as handle:
+            for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+                digest.update(chunk)
+        return digest.hexdigest()[:12]
+    return _stable_id(str(path))
+
+
+def _item_source_ref(item: ContentItem) -> str:
+    local_path = item.raw_meta.get("local_path") if isinstance(item.raw_meta, dict) else None
+    if isinstance(local_path, str) and local_path.strip():
+        return local_path
+    return item.url
+
+
+def _title_from_source_ref(source_ref: str) -> str:
+    parsed = urlparse(source_ref)
+    path = Path(parsed.path if parsed.scheme else source_ref)
+    return path.stem or "PDF document"
+
+
+def _stable_id(value: str) -> str:
+    return hashlib.sha256(value.strip().encode("utf-8")).hexdigest()[:12]

openmd/adapters/pdf/backends/__init__.py

@@ -0,0 +1 @@
+"""Optional document parser backends."""

openmd/adapters/pdf/backends/markitdown.py

@@ -0,0 +1,165 @@
+"""Optional local MarkItDown PDF/Office backend (legacy PDF backend name).
+
+The dependency is imported lazily so normal Openmd imports, mock lanes, and
+safe skeleton discovery never execute the external parser. The backend only
+parses local files or ``file://`` references; remote PDF fetching is deliberately
+out of scope for this local document lane.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import re
+from pathlib import Path
+from typing import Any
+from zipfile import BadZipFile, ZipFile
+
+from openmd.adapters.local_audio.adapter import local_path_from_input
+from openmd.adapters.pdf.parser import MissingPDFParser, PDFParser
+from openmd.core.models import TranscriptResult
+from openmd.core.detector import DOCUMENT_EXTENSIONS
+from openmd.core.secrets import sanitize_error_summary
+
+
+class MarkItDownPDFParser(PDFParser):
+    name = "pdf.markitdown"
+
+    def is_available(self) -> bool:
+        return importlib.util.find_spec("markitdown") is not None
+
+    def parse(self, source_ref: str, *, title: str) -> TranscriptResult:
+        source_path = _local_existing_path(source_ref)
+        if source_path is None:
+            return _failed_result(
+                self.name,
+                source_ref,
+                title,
+                "document_local_file_required",
+                "MarkItDown document parsing requires a local file path or file:// reference.",
+            )
+
+        if source_path.suffix.lower() not in DOCUMENT_EXTENSIONS:
+            return _failed_result(
+                self.name, str(source_path), title, "document_format_unsupported",
+                "Supported local document formats are PDF, DOCX, PPTX and XLSX.",
+            )
+
+        source_format = source_path.suffix.lower().lstrip(".")
+        office_parts = {"docx": "word/document.xml", "pptx": "ppt/presentation.xml", "xlsx": "xl/workbook.xml"}
+        if source_format in office_parts:
+            try:
+                with ZipFile(source_path) as archive:
+                    if not {"[Content_Types].xml", office_parts[source_format]}.issubset(archive.namelist()):
+                        raise BadZipFile("required Office parts are missing")
+            except (BadZipFile, OSError):
+                return _failed_result(
+                    self.name, str(source_path), title, "document_invalid_container",
+                    "The Office input is not a valid OOXML document container.",
+                )
+
+        if not self.is_available():
+            return MissingPDFParser(self.name).parse(str(source_path), title=title)
+
+        try:
+            from markitdown import MarkItDown
+
+            if source_format in office_parts:
+                from markitdown.converters import DocxConverter, PptxConverter, XlsxConverter
+
+                # Do not let a broken Office file fall back to plain text or a
+                # ZIP file listing and become a falsely verified document.
+                converter = {"docx": DocxConverter, "pptx": PptxConverter, "xlsx": XlsxConverter}[source_format]
+                parser = MarkItDown(enable_builtins=False)
+                parser.register_converter(converter())
+            else:
+                parser = MarkItDown()
+            converted: Any = parser.convert(str(source_path))
+            text = (getattr(converted, "text_content", None) or "").strip()
+        except Exception as exc:  # pragma: no cover - dependency-specific safety net
+            return _failed_result(
+                self.name,
+                str(source_path),
+                title,
+                "document_extraction_failed",
+                exc,
+            )
+
+        if not _has_document_text(text, source_format):
+            return _failed_result(
+                self.name,
+                str(source_path),
+                title,
+                "document_empty_text",
+                "MarkItDown returned no text for the document.",
+            )
+
+        document_title = title or source_path.stem or "PDF document"
+        return TranscriptResult(
+            text=_with_title(document_title, text),
+            status="verified",
+            source_level="native_text",
+            backend=self.name,
+            source_notes=[
+                f"MarkItDown parsed local {source_format.upper()} text.",
+                "Raw document file was referenced, not copied into the package.",
+                "Native text extraction does not certify reading order, tables, formulas or image OCR quality.",
+            ],
+            title=document_title,
+            metadata={
+                "profile": "document",
+                "parser_backend": self.name,
+                "source_ref": str(source_path),
+                "page_mapping": "not_available",
+                "source_format": source_format,
+            },
+        )
+
+
+def _has_document_text(text: str, source_format: str) -> bool:
+    """Ignore converter scaffolding without removing actual source headings."""
+    if source_format not in {"pptx", "xlsx"}:
+        return bool(text.strip())
+    visible = re.sub(r"<!--.*?-->", "", text, flags=re.DOTALL)
+    if source_format == "xlsx":
+        # Sheet headings and pandas' unnamed-column labels are generated even
+        # for empty workbooks, so they cannot establish source body coverage.
+        visible = re.sub(r"^## .*?$", "", visible, flags=re.MULTILINE)
+        visible = re.sub(r"Unnamed: \d+(?:_level_\d+)?", "", visible)
+        visible = re.sub(r"[|\s:\-]", "", visible)
+    return bool(visible.strip())
+
+
+def _local_existing_path(source_ref: str) -> Path | None:
+    path = local_path_from_input(source_ref)
+    if path is None:
+        return None
+    return path if path.is_file() else None
+
+
+def _with_title(title: str, text: str) -> str:
+    if text.lstrip().startswith("#"):
+        return text if text.endswith("\n") else f"{text}\n"
+    return f"# {title}\n\n{text}\n"
+
+
+def _failed_result(
+    backend: str,
+    source_ref: str,
+    title: str,
+    code: str,
+    error: object,
+) -> TranscriptResult:
+    return TranscriptResult(
+        text="",
+        status="failed",
+        source_level="partial_extraction",
+        backend=backend,
+        source_notes=[
+            "Document parser backend did not produce usable text.",
+            "Raw document file was referenced, not copied into the package.",
+        ],
+        error_code=code,
+        error_summary=sanitize_error_summary(error),
+        title=title or "PDF document",
+        metadata={"profile": "document", "parser_backend": backend, "source_ref": source_ref},
+    )

openmd/adapters/pdf/parser.py

@@ -0,0 +1,56 @@
+"""PDF parser backend abstractions."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from openmd.core.models import TranscriptResult
+from openmd.core.secrets import sanitize_error_summary
+
+
+class PDFParser(ABC):
+    name: str
+
+    @abstractmethod
+    def parse(self, source_ref: str, *, title: str) -> TranscriptResult:
+        raise NotImplementedError
+
+
+class MockPDFParser(PDFParser):
+    name = "pdf.mock"
+
+    def parse(self, source_ref: str, *, title: str) -> TranscriptResult:
+        return TranscriptResult(
+            text=f"# {title}\n\nPage 1: Mock PDF document text for Openmd package smoke.\n",
+            status="verified",
+            source_level="native_text",
+            backend=self.name,
+            source_notes=[
+                "PDF mock backend only",
+                "raw PDF file was referenced, not copied into the package",
+            ],
+            title=title or "PDF document",
+            metadata={"profile": "document", "parser_backend": self.name, "source_ref": source_ref},
+        )
+
+class MissingPDFParser(PDFParser):
+    def __init__(self, name: str) -> None:
+        self.name = name
+
+    def parse(self, source_ref: str, *, title: str) -> TranscriptResult:
+        return TranscriptResult(
+            text="",
+            status="failed",
+            source_level="partial_extraction",
+            backend=self.name,
+            source_notes=[
+                "PDF parser backend is not installed or not enabled",
+                "no external PDF parser command was attempted",
+            ],
+            error_code="document_backend_missing",
+            error_summary=sanitize_error_summary(
+                f"PDF parser backend {self.name!r} is not installed or enabled."
+            ),
+            title=title or "PDF document",
+            metadata={"profile": "document", "parser_backend": self.name, "source_ref": source_ref},
+        )

openmd/adapters/qq_group/__init__.py

@@ -0,0 +1,11 @@
+"""QQ group conversation adapter factory."""
+
+from __future__ import annotations
+
+from openmd.adapters.base import PlatformAdapter
+
+from .adapter import QQGroupAdapter
+
+
+def create_adapter(platform_id: str, default_backend: str) -> PlatformAdapter:
+    return QQGroupAdapter(platform_id, default_backend)

openmd/adapters/qq_group/adapter.py

@@ -0,0 +1,289 @@
+"""Mock/gated adapter for QQ group export package inputs."""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from pathlib import Path
+from typing import Any
+from urllib.parse import unquote, urlparse
+
+from openmd.adapters.base import PlatformAdapter
+from openmd.adapters.high_privacy import high_privacy_blocker_result
+from openmd.adapters.qq_group.parsers.html import parse_qq_html_export
+from openmd.adapters.qq_group.parsers.json import parse_qq_json_export
+from openmd.core.detector import detect_platform
+from openmd.core.models import ContentItem, TranscriptResult
+
+
+class QQGroupAdapter(PlatformAdapter):
+    """Resolve QQ group exports without database access or decryption."""
+
+    def __init__(self, platform_id: str = "qq_group", default_backend: str = "qq_group.export") -> None:
+        self.id = platform_id
+        self.default_backend = default_backend
+
+    def can_handle(self, input_value: str) -> bool:
+        return detect_platform(input_value) == self.id
+
+    def resolve_items(
+        self,
+        input_value: str,
+        *,
+        limit: int | None = None,
+        all: bool = False,
+    ) -> list[ContentItem]:
+        value = input_value.strip()
+        if _is_selector(value):
+            group_name = _group_from_selector(value)
+            raw_meta: dict[str, Any] = {
+                "profile": "conversation",
+                "permission_scope": "user_accessible",
+                "target_kind": "qq_group",
+                "group_name": group_name,
+                "discovery_limit": limit,
+                "allow_all": all,
+            }
+            title = f"QQ group: {group_name}"
+            identity = _stable_hash(group_name)
+        else:
+            path = _local_path_from_input(value)
+            group_name = _group_name_from_path(path)
+            raw_meta = {
+                "profile": "conversation",
+                "permission_scope": "user_export",
+                "target_kind": "qq_group_export",
+                "group_name": group_name,
+                "local_path": str(path),
+                "export_format": path.suffix.lower().lstrip("."),
+                "discovery_limit": limit,
+                "allow_all": all,
+            }
+            title = f"QQ group export: {group_name}"
+            identity = _stable_hash(str(path.resolve()) if path.exists() else str(path))
+
+        return [
+            ContentItem(
+                platform=self.id,
+                input_type="collection",
+                url=value,
+                id=identity,
+                title=title,
+                source_type="conversation",
+                raw_meta=raw_meta,
+            )
+        ]
+
+    def extract_text(
+        self,
+        item: ContentItem,
+        *,
+        backend: str | None = None,
+        backend_chain: tuple[str, ...] | None = None,
+        allow_fallback: bool = True,
+    ) -> TranscriptResult:
+        selected = (backend_chain or (backend or self.default_backend,))[0]
+        if selected in {"mock", "qq_group.mock"}:
+            messages = _messages_from_item(item)
+            return TranscriptResult(
+                text=render_qq_group_markdown(messages),
+                status="verified",
+                source_level="manual_export",
+                backend=selected,
+                title=item.title or "QQ group conversation",
+                source_notes=[
+                    "local mock adapter smoke only; no QQ database, session, DB key extraction, or decryption action was attempted",
+                ],
+                metadata={"profile": "conversation", "messages": messages},
+            )
+        if selected == "qq_group.export":
+            messages, blocker = _messages_from_export_item(item)
+            if blocker is not None:
+                return high_privacy_blocker_result(
+                    platform=self.id,
+                    backend=selected,
+                    target_kind=str(item.raw_meta.get("target_kind") or "qq_group_export"),
+                    state=blocker["state"],
+                    error_code=blocker["error_code"],
+                    error_summary=blocker["error_summary"],
+                    next_action=blocker["next_action"],
+                    permission_scope=str(item.raw_meta.get("permission_scope") or "unknown"),
+                    title=item.title or "QQ group conversation",
+                    source_notes=[
+                        "qq group adapter resolved the explicit selector/export only; no QQ database, DB key extraction, or decryption action was attempted",
+                        f"allow_fallback={allow_fallback}",
+                    ],
+                )
+            return TranscriptResult(
+                text=render_qq_group_markdown(messages),
+                status="verified",
+                source_level="manual_export",
+                backend=selected,
+                title=item.title or "QQ group conversation",
+                source_notes=[
+                    "parsed an explicitly provided QQ group export file",
+                    "no QQ database, DB key extraction, session, or decryption action was attempted",
+                ],
+                metadata={"profile": "conversation", "messages": messages},
+            )
+        return high_privacy_blocker_result(
+            platform=self.id,
+            backend=selected,
+            target_kind=str(item.raw_meta.get("target_kind") or "qq_group"),
+            state="export_required",
+            error_code="export_required",
+            error_summary="Provide an approved QQ group export file before running real parsing.",
+            next_action="Export the scoped QQ group conversation to JSON, HTML, or TXT and pass that local file to Openmd.",
+            permission_scope=str(item.raw_meta.get("permission_scope") or "unknown"),
+            title=item.title or "QQ group conversation",
+            source_notes=[
+                "qq group adapter resolved the export/selector only; no QQ database, DB key extraction, or decryption action was attempted",
+                f"allow_fallback={allow_fallback}",
+            ],
+        )
+
+
+def parse_qq_export_file(path: Path) -> list[dict[str, Any]]:
+    suffix = path.suffix.lower()
+    text = path.read_text(encoding="utf-8")
+    if suffix == ".json":
+        return parse_qq_json_export(text)
+    if suffix == ".html":
+        return parse_qq_html_export(text)
+    if suffix == ".txt":
+        return parse_qq_txt_export(text)
+    raise ValueError(f"unsupported QQ export suffix: {suffix}")
+
+
+def parse_qq_txt_export(text: str) -> list[dict[str, Any]]:
+    messages: list[dict[str, Any]] = []
+    pattern = re.compile(r"^\[?(?P<time>[^\]\s][^\]]{5,40})\]?\s+(?P<speaker>[^:：]+)[:：]\s*(?P<text>.*)$")
+    for index, line in enumerate(text.splitlines(), start=1):
+        stripped = line.strip()
+        if not stripped:
+            continue
+        match = pattern.match(stripped)
+        if match:
+            messages.append(
+                {
+                    "id": f"txt_{index:04d}",
+                    "timestamp": match.group("time").strip(),
+                    "speaker": match.group("speaker").strip(),
+                    "text": match.group("text").strip(),
+                    "attachments": [],
+                    "links": [],
+                    "source_map": [],
+                }
+            )
+    return messages
+
+
+def render_qq_group_markdown(messages: list[dict[str, Any]]) -> str:
+    lines = ["# QQ group conversation", ""]
+    for message in messages:
+        timestamp = str(message.get("timestamp") or "unknown-time")
+        speaker = str(message.get("speaker") or "unknown")
+        text = str(message.get("text") or "").strip() or "_No text content._"
+        lines.append(f"- [{timestamp}] {speaker}: {text}")
+    return "\n".join(lines).strip() + "\n"
+
+
+def _messages_from_item(item: ContentItem) -> list[dict[str, Any]]:
+    raw_meta = item.raw_meta if isinstance(item.raw_meta, dict) else {}
+    value = raw_meta.get("messages")
+    if isinstance(value, list) and value:
+        return [message for message in value if isinstance(message, dict)]
+
+    local_path = raw_meta.get("local_path")
+    if isinstance(local_path, str):
+        path = Path(local_path)
+        if path.exists():
+            messages = parse_qq_export_file(path)
+            if messages:
+                return messages
+
+    group_name = str(raw_meta.get("group_name") or "mock-group")
+    return [
+        {
+            "id": "qq_msg_001",
+            "timestamp": "2026-06-23T14:00:00+08:00",
+            "speaker": "Mock QQ sender",
+            "text": f"Mock message scoped to QQ group {group_name}.",
+            "attachments": [],
+            "links": [],
+            "source_map": [],
+        }
+    ]
+
+
+def _messages_from_export_item(item: ContentItem) -> tuple[list[dict[str, Any]], dict[str, str] | None]:
+    raw_meta = item.raw_meta if isinstance(item.raw_meta, dict) else {}
+    local_path = raw_meta.get("local_path")
+    if not isinstance(local_path, str) or not local_path.strip():
+        return [], {
+            "state": "export_required",
+            "error_code": "export_required",
+            "error_summary": "A scoped QQ group export file is required.",
+            "next_action": "Export the scoped QQ group conversation to JSON, HTML, or TXT and pass that local file to Openmd.",
+        }
+    path = Path(local_path)
+    if not path.exists():
+        return [], {
+            "state": "export_required",
+            "error_code": "export_required",
+            "error_summary": "The configured QQ group export file does not exist.",
+            "next_action": "Provide an existing scoped QQ group JSON, HTML, or TXT export file.",
+        }
+    try:
+        messages = parse_qq_export_file(path)
+    except ValueError as exc:
+        return [], {
+            "state": "export_required",
+            "error_code": "export_format_unsupported",
+            "error_summary": str(exc),
+            "next_action": "Use a supported scoped QQ export format: JSON, HTML, or TXT.",
+        }
+    if not messages:
+        return [], {
+            "state": "no_delta",
+            "error_code": "no_delta",
+            "error_summary": "The QQ group export file contained no parseable messages.",
+            "next_action": "Confirm the export scope/date range and provide a non-empty scoped export.",
+        }
+    return messages, None
+
+
+def _is_selector(value: str) -> bool:
+    parsed = urlparse(value)
+    return parsed.scheme in {"qq", "qq-group", "qq_export", "qq-export"}
+
+
+def _group_from_selector(value: str) -> str:
+    parsed = urlparse(value)
+    if parsed.scheme == "qq" and parsed.netloc.lower() == "group":
+        group_name = parsed.path.strip("/")
+    else:
+        group_name = parsed.netloc or parsed.path.strip("/")
+    group_name = unquote(group_name)
+    if not group_name or group_name in {"*", "all"}:
+        raise ValueError("QQ group selector must name one explicit group.")
+    return group_name
+
+
+def _local_path_from_input(value: str) -> Path:
+    parsed = urlparse(value)
+    if parsed.scheme == "file":
+        return Path(unquote(parsed.path))
+    return Path(value)
+
+
+def _group_name_from_path(path: Path) -> str:
+    stem = path.name
+    for suffix in (".json", ".html", ".txt"):
+        if stem.lower().endswith(suffix):
+            stem = stem[: -len(suffix)]
+    return stem or "qq-group-export"
+
+
+def _stable_hash(value: str) -> str:
+    return hashlib.sha256(value.encode("utf-8")).hexdigest()[:12]