openmd-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- openmd/__init__.py +3 -0
- openmd/__main__.py +5 -0
- openmd/adapters/__init__.py +1 -0
- openmd/adapters/base.py +95 -0
- openmd/adapters/bilibili/__init__.py +11 -0
- openmd/adapters/bilibili/adapter.py +104 -0
- openmd/adapters/bilibili/discovery.py +33 -0
- openmd/adapters/douyin/__init__.py +11 -0
- openmd/adapters/douyin/adapter.py +117 -0
- openmd/adapters/douyin/discovery.py +36 -0
- openmd/adapters/douyin/sidecar.py +77 -0
- openmd/adapters/generic_media/__init__.py +14 -0
- openmd/adapters/high_privacy.py +77 -0
- openmd/adapters/local_audio/__init__.py +14 -0
- openmd/adapters/local_audio/adapter.py +169 -0
- openmd/adapters/local_audio/asr.py +58 -0
- openmd/adapters/local_audio/backends/__init__.py +1 -0
- openmd/adapters/local_audio/backends/auto.py +177 -0
- openmd/adapters/local_audio/backends/faster_whisper.py +285 -0
- openmd/adapters/local_audio/backends/funasr_sense_voice.py +14 -0
- openmd/adapters/local_audio/backends/qwen3_asr.py +254 -0
- openmd/adapters/opencli.py +770 -0
- openmd/adapters/pdf/__init__.py +11 -0
- openmd/adapters/pdf/adapter.py +153 -0
- openmd/adapters/pdf/backends/__init__.py +1 -0
- openmd/adapters/pdf/backends/markitdown.py +165 -0
- openmd/adapters/pdf/parser.py +56 -0
- openmd/adapters/qq_group/__init__.py +11 -0
- openmd/adapters/qq_group/adapter.py +289 -0
- openmd/adapters/qq_group/parsers/__init__.py +1 -0
- openmd/adapters/qq_group/parsers/html.py +60 -0
- openmd/adapters/qq_group/parsers/json.py +71 -0
- openmd/adapters/registry.py +149 -0
- openmd/adapters/screenshot/__init__.py +11 -0
- openmd/adapters/screenshot/adapter.py +154 -0
- openmd/adapters/screenshot/backends/__init__.py +1 -0
- openmd/adapters/screenshot/backends/paddle.py +15 -0
- openmd/adapters/screenshot/backends/rapidocr.py +176 -0
- openmd/adapters/screenshot/backends/tesseract.py +15 -0
- openmd/adapters/screenshot/ocr.py +84 -0
- openmd/adapters/wechat_group/__init__.py +11 -0
- openmd/adapters/wechat_group/adapter.py +160 -0
- openmd/adapters/wechat_group/wx_cli.py +268 -0
- openmd/adapters/wechat_moments/__init__.py +11 -0
- openmd/adapters/wechat_moments/adapter.py +167 -0
- openmd/adapters/wechat_moments/wx_cli.py +265 -0
- openmd/adapters/weixin/__init__.py +11 -0
- openmd/adapters/weixin/adapter.py +148 -0
- openmd/adapters/weixin/discovery.py +510 -0
- openmd/adapters/weixin/report.py +278 -0
- openmd/adapters/xiaohongshu.py +158 -0
- openmd/backends/__init__.py +1 -0
- openmd/backends/base.py +204 -0
- openmd/backends/bilibili.py +1480 -0
- openmd/backends/bilibili_wbi.py +277 -0
- openmd/backends/discourse.py +426 -0
- openmd/backends/douyin.py +868 -0
- openmd/backends/douyin_detail.py +124 -0
- openmd/backends/douyin_providers.py +602 -0
- openmd/backends/executor.py +546 -0
- openmd/backends/feishu.py +861 -0
- openmd/backends/generic_media.py +77 -0
- openmd/backends/interactive.py +60 -0
- openmd/backends/ocr.py +41 -0
- openmd/backends/opencli.py +82 -0
- openmd/backends/pdf.py +122 -0
- openmd/backends/registry.py +234 -0
- openmd/backends/screenshot.py +79 -0
- openmd/backends/video_metadata.py +167 -0
- openmd/backends/web_article.py +363 -0
- openmd/backends/wechat_group.py +35 -0
- openmd/backends/wechat_moments.py +35 -0
- openmd/backends/weixin.py +767 -0
- openmd/backends/x.py +455 -0
- openmd/backends/xiaoe.py +940 -0
- openmd/backends/xiaohongshu.py +911 -0
- openmd/backends/xiaohongshu_providers.py +876 -0
- openmd/backends/xiaoyuzhou.py +1017 -0
- openmd/backends/youtube.py +280 -0
- openmd/backends/youtube_discovery.py +1245 -0
- openmd/backends/youtube_transcript.py +213 -0
- openmd/browser_extension/background.js +231 -0
- openmd/browser_extension/icons/icon-128.png +0 -0
- openmd/browser_extension/icons/icon-16.png +0 -0
- openmd/browser_extension/icons/icon-32.png +0 -0
- openmd/browser_extension/icons/icon-48.png +0 -0
- openmd/browser_extension/icons/icon-512.png +0 -0
- openmd/browser_extension/idle.html +6 -0
- openmd/browser_extension/manifest.json +43 -0
- openmd/browser_extension/options.html +42 -0
- openmd/browser_extension/options.js +117 -0
- openmd/browser_extension/popup.css +71 -0
- openmd/browser_extension/popup.html +17 -0
- openmd/browser_extension/popup.js +59 -0
- openmd/browser_extension/vendor/opencli-bridge/LICENSE +190 -0
- openmd/browser_extension/vendor/opencli-bridge/VENDOR.md +31 -0
- openmd/browser_extension/vendor/opencli-bridge/background.js +2742 -0
- openmd/browser_host.py +503 -0
- openmd/browser_setup.py +106 -0
- openmd/cli.py +2012 -0
- openmd/console.py +32 -0
- openmd/core/__init__.py +1 -0
- openmd/core/capabilities.py +162 -0
- openmd/core/content_validation.py +323 -0
- openmd/core/coverage.py +163 -0
- openmd/core/detector.py +247 -0
- openmd/core/doctor.py +104 -0
- openmd/core/execution_policy.py +164 -0
- openmd/core/filename.py +64 -0
- openmd/core/input_validation.py +91 -0
- openmd/core/local_config.py +96 -0
- openmd/core/manifest.py +156 -0
- openmd/core/models.py +168 -0
- openmd/core/opencli_output.py +34 -0
- openmd/core/operation_contract.py +60 -0
- openmd/core/output.py +194 -0
- openmd/core/package_contract.py +335 -0
- openmd/core/package_validator.py +489 -0
- openmd/core/package_writer.py +666 -0
- openmd/core/planner.py +345 -0
- openmd/core/preferences.py +61 -0
- openmd/core/readable.py +129 -0
- openmd/core/receipt.py +285 -0
- openmd/core/renderer.py +185 -0
- openmd/core/runlog.py +208 -0
- openmd/core/secrets.py +313 -0
- openmd/doctor_view.py +68 -0
- openmd/interactive.py +453 -0
- openmd/ocr/__init__.py +1 -0
- openmd/ocr/breakers.py +84 -0
- openmd/ocr/config.py +54 -0
- openmd/ocr/documents.py +522 -0
- openmd/ocr/endpoint.py +72 -0
- openmd/ocr/layout.py +92 -0
- openmd/ocr/layout_filter.py +118 -0
- openmd/ocr/layout_worker.py +89 -0
- openmd/ocr/native_math.py +220 -0
- openmd/ocr/office.py +305 -0
- openmd/ocr/ollama.py +200 -0
- openmd/ocr/otsl.py +51 -0
- openmd/ocr/ovis-prompt.txt +1 -0
- openmd/ocr/pipeline.py +384 -0
- openmd/ocr/settings.json +25 -0
- openmd/ocr/tiling.py +631 -0
- openmd/post_video.py +154 -0
- openmd/probe.py +397 -0
- openmd/routing/__init__.py +1 -0
- openmd/routing/cards/builtin.json +620 -0
- openmd/routing/cards/defuddle.json +70 -0
- openmd/routing/cards/x-tweet-fetcher.json +71 -0
- openmd/routing/cards/xyz-dl.json +60 -0
- openmd/routing/cards/yutto.json +68 -0
- openmd/routing/entry_shim.py +71 -0
- openmd/routing/lifecycle.py +459 -0
- openmd/routing/loader.py +138 -0
- openmd/routing/providers.py +158 -0
- openmd/routing/readers.py +108 -0
- openmd/routing/routes.json +504 -0
- openmd/routing/runner.py +192 -0
- openmd/routing/view.py +40 -0
- openmd/runtime/__init__.py +1 -0
- openmd/runtime/_windows_job.py +234 -0
- openmd/runtime/asr.py +217 -0
- openmd/runtime/asr_worker.py +119 -0
- openmd/runtime/gpu_lock.py +81 -0
- openmd/runtime/gpu_quarantine.py +85 -0
- openmd/runtime/interpreters.py +407 -0
- openmd/runtime/managed_tools.py +204 -0
- openmd/runtime/models.py +202 -0
- openmd/runtime/services.py +175 -0
- openmd/runtime/tempdirs.py +227 -0
- openmd_cli-0.1.0.dist-info/METADATA +168 -0
- openmd_cli-0.1.0.dist-info/RECORD +177 -0
- openmd_cli-0.1.0.dist-info/WHEEL +5 -0
- openmd_cli-0.1.0.dist-info/entry_points.txt +2 -0
- openmd_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
- openmd_cli-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""Runtime providers for long screenshot OCR inputs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from openmd.adapters.screenshot.adapter import ScreenshotOcrAdapter
|
|
6
|
+
from openmd.adapters.screenshot.backends.rapidocr import RapidOCRBackend
|
|
7
|
+
from openmd.adapters.screenshot.ocr import MissingOCRBackend, MockOCRBackend
|
|
8
|
+
from openmd.backends.base import (
|
|
9
|
+
BackendCapability,
|
|
10
|
+
BackendDescriptor,
|
|
11
|
+
BackendExtractRequest,
|
|
12
|
+
BackendHealth,
|
|
13
|
+
)
|
|
14
|
+
from openmd.backends.executor import BackendProviderRegistry
|
|
15
|
+
from openmd.backends.registry import BackendRuntimeOptions
|
|
16
|
+
from openmd.core.models import TranscriptResult
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class ScreenshotOCRProvider:
|
|
20
|
+
def __init__(self, backend) -> None:
|
|
21
|
+
self.backend = backend
|
|
22
|
+
self.name = backend.name
|
|
23
|
+
|
|
24
|
+
def descriptor(self) -> BackendDescriptor:
|
|
25
|
+
source_level = "ocr" if isinstance(self.backend, MockOCRBackend) else "partial_extraction"
|
|
26
|
+
return BackendDescriptor(
|
|
27
|
+
name=self.name,
|
|
28
|
+
layer="local",
|
|
29
|
+
capabilities=(
|
|
30
|
+
BackendCapability(
|
|
31
|
+
platforms=("screenshot",),
|
|
32
|
+
source_types=("image_text",),
|
|
33
|
+
input_types=("single", "batch", "collection", "generic"),
|
|
34
|
+
source_levels=(source_level,),
|
|
35
|
+
requires_network=False,
|
|
36
|
+
requires_credentials=False,
|
|
37
|
+
),
|
|
38
|
+
),
|
|
39
|
+
description="Long screenshot OCR provider.",
|
|
40
|
+
provider="openmd.screenshot",
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
def health(self) -> BackendHealth:
|
|
44
|
+
is_available = getattr(self.backend, "is_available", None)
|
|
45
|
+
if callable(is_available) and not is_available():
|
|
46
|
+
return BackendHealth(
|
|
47
|
+
ok=False,
|
|
48
|
+
error_code="ocr_backend_missing",
|
|
49
|
+
error_summary=f"{self.name} is not installed or enabled.",
|
|
50
|
+
)
|
|
51
|
+
if isinstance(self.backend, MissingOCRBackend):
|
|
52
|
+
return BackendHealth(
|
|
53
|
+
ok=False,
|
|
54
|
+
error_code="ocr_backend_missing",
|
|
55
|
+
error_summary=f"{self.name} is not installed or enabled.",
|
|
56
|
+
)
|
|
57
|
+
return BackendHealth(ok=True, dependencies={self.name: "ok"})
|
|
58
|
+
|
|
59
|
+
def extract(self, request: BackendExtractRequest) -> TranscriptResult:
|
|
60
|
+
adapter = ScreenshotOcrAdapter("screenshot", request.primary_backend)
|
|
61
|
+
return adapter.extract_text(
|
|
62
|
+
request.item,
|
|
63
|
+
backend=request.primary_backend,
|
|
64
|
+
backend_chain=(request.primary_backend,),
|
|
65
|
+
allow_fallback=False,
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def register_backend_providers(
|
|
70
|
+
registry: BackendProviderRegistry,
|
|
71
|
+
options: BackendRuntimeOptions,
|
|
72
|
+
) -> None:
|
|
73
|
+
from openmd.backends.ocr import register_ocr_providers
|
|
74
|
+
|
|
75
|
+
register_ocr_providers(registry, options)
|
|
76
|
+
registry.register("screenshot.mock", ScreenshotOCRProvider(MockOCRBackend()))
|
|
77
|
+
registry.register("ocr.rapidocr", ScreenshotOCRProvider(RapidOCRBackend()))
|
|
78
|
+
registry.register("ocr.paddle", ScreenshotOCRProvider(MissingOCRBackend("ocr.paddle")))
|
|
79
|
+
registry.register("ocr.tesseract", ScreenshotOCRProvider(MissingOCRBackend("ocr.tesseract")))
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"""Optional public video metadata, separate from transcript acquisition.
|
|
2
|
+
|
|
3
|
+
Only the explicit public-network gate permits a request. Browser permission
|
|
4
|
+
alone never enables these requests, and metadata failure cannot lose text.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import re
|
|
11
|
+
from collections.abc import Callable
|
|
12
|
+
from dataclasses import replace
|
|
13
|
+
from datetime import datetime, timezone
|
|
14
|
+
from typing import Any
|
|
15
|
+
from urllib.parse import urlencode
|
|
16
|
+
from urllib.request import Request, urlopen
|
|
17
|
+
|
|
18
|
+
from openmd.core.filename import short_hash
|
|
19
|
+
from openmd.core.models import ContentItem, TranscriptResult
|
|
20
|
+
|
|
21
|
+
from .youtube_transcript import extract_youtube_video_id
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
MetadataFetcher = Callable[[str, int], Any]
|
|
25
|
+
_YOUTUBE_ID = re.compile(r"^[A-Za-z0-9_-]{11}$")
|
|
26
|
+
_BILIBILI_ID = re.compile(r"BV[A-Za-z0-9]{10}")
|
|
27
|
+
_PLACEHOLDER_TITLES = frozenset({"untitled", "unknown", "video", "audio"})
|
|
28
|
+
_MAX_RESPONSE_BYTES = 512 * 1024
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _value(value: object, *, limit: int = 512) -> str | None:
|
|
32
|
+
if not isinstance(value, str):
|
|
33
|
+
return None
|
|
34
|
+
cleaned = " ".join(value.split()).strip()
|
|
35
|
+
return cleaned[:limit] if cleaned else None
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _identity(item: ContentItem, result: TranscriptResult) -> tuple[str, str | None]:
|
|
39
|
+
if item.platform == "youtube":
|
|
40
|
+
for candidate in (item.id, result.metadata.get("video_id"), extract_youtube_video_id(item.url)):
|
|
41
|
+
if isinstance(candidate, str) and _YOUTUBE_ID.fullmatch(candidate):
|
|
42
|
+
return candidate, candidate
|
|
43
|
+
else:
|
|
44
|
+
for candidate in (item.id, result.metadata.get("bvid"), item.url):
|
|
45
|
+
match = _BILIBILI_ID.search(candidate) if isinstance(candidate, str) else None
|
|
46
|
+
if match:
|
|
47
|
+
return match.group(), match.group()
|
|
48
|
+
return short_hash(item.url), None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _source_title(value: object, *, identifier: str, platform: str) -> str | None:
|
|
52
|
+
title = _value(value)
|
|
53
|
+
if title is None or title.casefold() in _PLACEHOLDER_TITLES:
|
|
54
|
+
return None
|
|
55
|
+
if title.casefold() in {identifier.casefold(), f"{platform} · {identifier}".casefold()}:
|
|
56
|
+
return None
|
|
57
|
+
return title
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _fetch_public_json(url: str, timeout: int) -> Any:
|
|
61
|
+
# These fixed public endpoints do not receive session state or credentials.
|
|
62
|
+
request = Request(url, headers={"User-Agent": "Openmd/0.0", "Accept": "application/json"})
|
|
63
|
+
with urlopen(request, timeout=timeout) as response:
|
|
64
|
+
if not 200 <= int(getattr(response, "status", 200)) < 300:
|
|
65
|
+
raise ValueError("metadata_http_error")
|
|
66
|
+
raw = response.read(_MAX_RESPONSE_BYTES + 1)
|
|
67
|
+
if len(raw) > _MAX_RESPONSE_BYTES:
|
|
68
|
+
raise ValueError("metadata_response_too_large")
|
|
69
|
+
return json.loads(raw.decode("utf-8-sig"))
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _public_metadata(platform: str, identifier: str, fetch: MetadataFetcher, timeout: int) -> dict[str, str | None]:
|
|
73
|
+
if platform == "youtube":
|
|
74
|
+
url = "https://www.youtube.com/oembed?" + urlencode({
|
|
75
|
+
"url": f"https://www.youtube.com/watch?v={identifier}", "format": "json",
|
|
76
|
+
})
|
|
77
|
+
payload = fetch(url, timeout)
|
|
78
|
+
if not isinstance(payload, dict):
|
|
79
|
+
raise ValueError("metadata_payload_invalid")
|
|
80
|
+
return {
|
|
81
|
+
"title": _value(payload.get("title")),
|
|
82
|
+
"author": _value(payload.get("author_name"), limit=256),
|
|
83
|
+
"published_at": None, # oEmbed does not establish a publication date.
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
payload = fetch("https://api.bilibili.com/x/web-interface/view?" + urlencode({"bvid": identifier}), timeout)
|
|
87
|
+
if not isinstance(payload, dict) or payload.get("code") not in (0, "0"):
|
|
88
|
+
raise ValueError("metadata_payload_invalid")
|
|
89
|
+
data = payload.get("data")
|
|
90
|
+
if not isinstance(data, dict):
|
|
91
|
+
raise ValueError("metadata_payload_invalid")
|
|
92
|
+
owner = data.get("owner")
|
|
93
|
+
pubdate = data.get("pubdate")
|
|
94
|
+
published_at = None
|
|
95
|
+
if isinstance(pubdate, (int, float)) and not isinstance(pubdate, bool) and pubdate > 0:
|
|
96
|
+
try:
|
|
97
|
+
published_at = datetime.fromtimestamp(pubdate, timezone.utc).date().isoformat()
|
|
98
|
+
except (OverflowError, OSError, ValueError):
|
|
99
|
+
pass
|
|
100
|
+
return {
|
|
101
|
+
"title": _value(data.get("title")),
|
|
102
|
+
"author": _value(owner.get("name"), limit=256) if isinstance(owner, dict) else None,
|
|
103
|
+
"published_at": published_at,
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def enrich_video_result(
|
|
108
|
+
item: ContentItem,
|
|
109
|
+
result: TranscriptResult,
|
|
110
|
+
*,
|
|
111
|
+
network_enabled: bool = False,
|
|
112
|
+
fetch_json: MetadataFetcher | None = None,
|
|
113
|
+
timeout: int = 8,
|
|
114
|
+
) -> TranscriptResult:
|
|
115
|
+
"""Fill absent video title/author, preserving all extraction evidence.
|
|
116
|
+
|
|
117
|
+
At most one bounded public request is made per call, only for a usable
|
|
118
|
+
YouTube/Bilibili result missing title or author and with network enabled.
|
|
119
|
+
Existing source metadata takes priority. No video title is inferred from
|
|
120
|
+
transcript words. ``fetch_json`` is an injectable JSON transport for local
|
|
121
|
+
verification; it obeys the same gate as the default transport.
|
|
122
|
+
"""
|
|
123
|
+
if item.platform not in {"youtube", "bilibili"} or not result.has_usable_text or result.backend == "mock":
|
|
124
|
+
return result
|
|
125
|
+
identifier, fetch_identifier = _identity(item, result)
|
|
126
|
+
title = _source_title(result.title, identifier=identifier, platform=item.platform) or _source_title(
|
|
127
|
+
item.title, identifier=identifier, platform=item.platform,
|
|
128
|
+
)
|
|
129
|
+
author = _value(result.author, limit=256) or _value(item.author, limit=256)
|
|
130
|
+
published_at = result.published_at or item.published_at
|
|
131
|
+
status = "already_present" if title and author else "not_requested"
|
|
132
|
+
source = None
|
|
133
|
+
if not (title and author) and network_enabled and fetch_identifier:
|
|
134
|
+
try:
|
|
135
|
+
public = _public_metadata(item.platform, fetch_identifier, fetch_json or _fetch_public_json, timeout)
|
|
136
|
+
title = title or _source_title(public["title"], identifier=identifier, platform=item.platform)
|
|
137
|
+
author = author or public["author"]
|
|
138
|
+
published_at = published_at or public["published_at"]
|
|
139
|
+
status = "available" if title and author else "partial"
|
|
140
|
+
source = "youtube.oembed" if item.platform == "youtube" else "bilibili.public_view"
|
|
141
|
+
except Exception:
|
|
142
|
+
# A good transcript remains a good transcript even when metadata
|
|
143
|
+
# is blocked, times out, or has an unexpected shape. Never retain
|
|
144
|
+
# upstream payloads, URLs, headers or exception text in this note.
|
|
145
|
+
status = "unavailable"
|
|
146
|
+
missing = [field for field, value in (("title", title), ("author", author)) if not value]
|
|
147
|
+
notes = list(result.source_notes)
|
|
148
|
+
if not title:
|
|
149
|
+
label = "YouTube" if item.platform == "youtube" else "Bilibili"
|
|
150
|
+
title = f"{label} · {identifier}"
|
|
151
|
+
note = "视频标题未取得;保留已提取文字,以平台和视频 ID 命名。"
|
|
152
|
+
if note not in notes:
|
|
153
|
+
notes.append(note)
|
|
154
|
+
elif not author:
|
|
155
|
+
note = "视频作者未取得;保留已提取文字。"
|
|
156
|
+
if note not in notes:
|
|
157
|
+
notes.append(note)
|
|
158
|
+
return replace(
|
|
159
|
+
result,
|
|
160
|
+
title=title,
|
|
161
|
+
author=author,
|
|
162
|
+
published_at=published_at,
|
|
163
|
+
source_notes=notes,
|
|
164
|
+
metadata={**result.metadata, "video_metadata": {
|
|
165
|
+
"status": status, "source": source, "missing_fields": missing,
|
|
166
|
+
}},
|
|
167
|
+
)
|
|
@@ -0,0 +1,363 @@
|
|
|
1
|
+
"""Generic web article extraction providers.
|
|
2
|
+
|
|
3
|
+
The primary provider wraps ``trafilatura`` as an optional local library. It is
|
|
4
|
+
registered only after ``--enable-network-backends`` because it fetches remote
|
|
5
|
+
pages. Browser/session fallbacks such as OpenCLI stay behind
|
|
6
|
+
``--enable-browser-backends``.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import shutil
|
|
12
|
+
import subprocess
|
|
13
|
+
from collections.abc import Callable
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from openmd.backends.base import (
|
|
17
|
+
BackendCapability,
|
|
18
|
+
BackendDescriptor,
|
|
19
|
+
BackendExtractRequest,
|
|
20
|
+
BackendHealth,
|
|
21
|
+
)
|
|
22
|
+
from openmd.backends.executor import BackendProviderRegistry
|
|
23
|
+
from openmd.backends.registry import BackendRuntimeOptions
|
|
24
|
+
from openmd.core.models import TranscriptResult
|
|
25
|
+
from openmd.core.secrets import sanitize_error_summary
|
|
26
|
+
from openmd.runtime.interpreters import run_interruptibly
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
WEB_ARTICLE_TRAFILATURA_BACKEND = "web_article.trafilatura"
|
|
30
|
+
OPENCLI_WEB_READ_BACKEND = "opencli.web.read"
|
|
31
|
+
CRAWL4AI_WEB_ARTICLE_BACKEND = "crawl4ai.web_article"
|
|
32
|
+
DEFAULT_MIN_ARTICLE_CHARS = 40
|
|
33
|
+
|
|
34
|
+
CommandRunner = Callable[[list[str], int], tuple[int, str, str]]
|
|
35
|
+
TrafilaturaFetcher = Callable[[str], object]
|
|
36
|
+
TrafilaturaExtractor = Callable[..., str | None]
|
|
37
|
+
TrafilaturaMetadataExtractor = Callable[[object], object]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def register_backend_providers(
|
|
41
|
+
registry: BackendProviderRegistry,
|
|
42
|
+
options: BackendRuntimeOptions,
|
|
43
|
+
) -> None:
|
|
44
|
+
"""Register web article providers behind explicit runtime gates."""
|
|
45
|
+
requested = options.backend_override
|
|
46
|
+
from openmd.backends.discourse import DISCOURSE_BACKEND, DiscourseProvider
|
|
47
|
+
|
|
48
|
+
if (options.enable_network_backends or options.enable_browser_backends) and requested in {None, DISCOURSE_BACKEND}:
|
|
49
|
+
registry.register(DISCOURSE_BACKEND, DiscourseProvider(options))
|
|
50
|
+
if options.enable_network_backends and requested in {None, WEB_ARTICLE_TRAFILATURA_BACKEND}:
|
|
51
|
+
registry.register(WEB_ARTICLE_TRAFILATURA_BACKEND, WebArticleTrafilaturaProvider())
|
|
52
|
+
|
|
53
|
+
if options.enable_browser_backends and requested in {None, OPENCLI_WEB_READ_BACKEND}:
|
|
54
|
+
registry.register(OPENCLI_WEB_READ_BACKEND, OpenCliWebReadProvider())
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class WebArticleTrafilaturaProvider:
|
|
58
|
+
"""Extract generic static articles through the optional trafilatura API."""
|
|
59
|
+
|
|
60
|
+
name = WEB_ARTICLE_TRAFILATURA_BACKEND
|
|
61
|
+
|
|
62
|
+
def __init__(
|
|
63
|
+
self,
|
|
64
|
+
*,
|
|
65
|
+
fetcher: TrafilaturaFetcher | None = None,
|
|
66
|
+
extractor: TrafilaturaExtractor | None = None,
|
|
67
|
+
metadata_extractor: TrafilaturaMetadataExtractor | None = None,
|
|
68
|
+
min_article_chars: int = DEFAULT_MIN_ARTICLE_CHARS,
|
|
69
|
+
) -> None:
|
|
70
|
+
self._fetcher = fetcher
|
|
71
|
+
self._extractor = extractor
|
|
72
|
+
self._metadata_extractor = metadata_extractor
|
|
73
|
+
self.min_article_chars = max(0, min_article_chars)
|
|
74
|
+
|
|
75
|
+
def descriptor(self) -> BackendDescriptor:
|
|
76
|
+
capability = BackendCapability(
|
|
77
|
+
platforms=("web_article",),
|
|
78
|
+
source_types=("article",),
|
|
79
|
+
input_types=("single", "batch"),
|
|
80
|
+
source_levels=("native_text", "partial_extraction"),
|
|
81
|
+
requires_network=True,
|
|
82
|
+
requires_credentials=False,
|
|
83
|
+
optional_config=("trafilatura",),
|
|
84
|
+
)
|
|
85
|
+
return BackendDescriptor(
|
|
86
|
+
name=self.name,
|
|
87
|
+
layer="L1 specialized extractor",
|
|
88
|
+
capabilities=(capability,),
|
|
89
|
+
description="Fetch and extract static web articles with optional local trafilatura.",
|
|
90
|
+
provider="openmd.backends.web_article",
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
def health(self) -> BackendHealth:
|
|
94
|
+
if self._fetcher is not None and self._extractor is not None:
|
|
95
|
+
return BackendHealth(ok=True, dependencies={"trafilatura": "injected"})
|
|
96
|
+
try:
|
|
97
|
+
import trafilatura # type: ignore[import-not-found] # noqa: F401
|
|
98
|
+
except ModuleNotFoundError:
|
|
99
|
+
return BackendHealth(
|
|
100
|
+
ok=False,
|
|
101
|
+
dependencies={"trafilatura": "missing"},
|
|
102
|
+
error_code="backend_dependency_missing",
|
|
103
|
+
error_summary="Install trafilatura or use a browser fallback backend.",
|
|
104
|
+
)
|
|
105
|
+
return BackendHealth(ok=True, dependencies={"trafilatura": "available"})
|
|
106
|
+
|
|
107
|
+
def extract(self, request: BackendExtractRequest) -> TranscriptResult:
|
|
108
|
+
item = request.item
|
|
109
|
+
if item.platform != "web_article":
|
|
110
|
+
return _failed(
|
|
111
|
+
self.name,
|
|
112
|
+
"unsupported_platform",
|
|
113
|
+
f"trafilatura web article provider only supports web_article, got {item.platform}.",
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
try:
|
|
117
|
+
fetcher, extractor, metadata_extractor = self._resolve_api()
|
|
118
|
+
except ModuleNotFoundError:
|
|
119
|
+
return _failed(
|
|
120
|
+
self.name,
|
|
121
|
+
"backend_dependency_missing",
|
|
122
|
+
"trafilatura is not installed; install it or enable browser fallback.",
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
try:
|
|
126
|
+
downloaded = fetcher(item.url)
|
|
127
|
+
except Exception as exc:
|
|
128
|
+
downloaded, fetch_error = None, exc
|
|
129
|
+
else:
|
|
130
|
+
fetch_error = None
|
|
131
|
+
if downloaded is None and self._fetcher is None:
|
|
132
|
+
# trafilatura's urllib3 client ignores system/SOCKS proxy settings;
|
|
133
|
+
# stdlib urllib honors HTTP(S)_PROXY and the Windows proxy.
|
|
134
|
+
try:
|
|
135
|
+
downloaded = _urllib_fetch(item.url)
|
|
136
|
+
except Exception as exc: # network errors are reported below
|
|
137
|
+
fetch_error = fetch_error or exc
|
|
138
|
+
if downloaded is None:
|
|
139
|
+
if fetch_error is not None:
|
|
140
|
+
return _failed(self.name, "web_article_fetch_failed", fetch_error)
|
|
141
|
+
return _failed(self.name, "web_article_fetch_empty", "trafilatura.fetch_url returned no document.")
|
|
142
|
+
|
|
143
|
+
try:
|
|
144
|
+
markdown = extractor(
|
|
145
|
+
downloaded,
|
|
146
|
+
output_format="markdown",
|
|
147
|
+
# Metadata is read separately below; True would embed a YAML
|
|
148
|
+
# block inside the body, duplicating the rendered front matter.
|
|
149
|
+
with_metadata=False,
|
|
150
|
+
include_images=True,
|
|
151
|
+
include_links=True,
|
|
152
|
+
url=item.url,
|
|
153
|
+
)
|
|
154
|
+
except Exception as exc:
|
|
155
|
+
return _failed(self.name, "web_article_extract_failed", exc)
|
|
156
|
+
|
|
157
|
+
text = (markdown or "").strip()
|
|
158
|
+
if len("".join(text.split())) < self.min_article_chars:
|
|
159
|
+
return _failed(
|
|
160
|
+
self.name,
|
|
161
|
+
"web_article_empty_content",
|
|
162
|
+
"trafilatura returned no usable article body.",
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
metadata = _metadata_dict(metadata_extractor, downloaded)
|
|
166
|
+
return TranscriptResult(
|
|
167
|
+
text=text.rstrip() + "\n",
|
|
168
|
+
status="verified",
|
|
169
|
+
source_level="native_text",
|
|
170
|
+
backend=self.name,
|
|
171
|
+
title=_metadata_value(metadata, "title"),
|
|
172
|
+
author=_metadata_value(metadata, "author"),
|
|
173
|
+
published_at=_metadata_value(metadata, "date", "published", "published_at"),
|
|
174
|
+
source_notes=[
|
|
175
|
+
"trafilatura extracted article Markdown locally from fetched HTML.",
|
|
176
|
+
"include_images=True and include_links=True were requested.",
|
|
177
|
+
"No browser, cloud API, media download, or ASR action was attempted.",
|
|
178
|
+
],
|
|
179
|
+
metadata={"strategy": "trafilatura_api"},
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
def _resolve_api(
|
|
183
|
+
self,
|
|
184
|
+
) -> tuple[TrafilaturaFetcher, TrafilaturaExtractor, TrafilaturaMetadataExtractor | None]:
|
|
185
|
+
if self._fetcher is not None and self._extractor is not None:
|
|
186
|
+
return self._fetcher, self._extractor, self._metadata_extractor
|
|
187
|
+
|
|
188
|
+
from trafilatura import extract, extract_metadata, fetch_url # type: ignore[import-not-found]
|
|
189
|
+
|
|
190
|
+
return fetch_url, extract, extract_metadata
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
class OpenCliWebReadProvider:
|
|
194
|
+
"""Browser/session fallback for generic web pages through OpenCLI."""
|
|
195
|
+
|
|
196
|
+
name = OPENCLI_WEB_READ_BACKEND
|
|
197
|
+
|
|
198
|
+
def __init__(self, *, runner: CommandRunner | None = None, timeout: int = 180) -> None:
|
|
199
|
+
self._runner = runner or _run_command
|
|
200
|
+
self.timeout = timeout
|
|
201
|
+
|
|
202
|
+
def descriptor(self) -> BackendDescriptor:
|
|
203
|
+
capability = BackendCapability(
|
|
204
|
+
platforms=("web_article", "weixin"),
|
|
205
|
+
source_types=("article",),
|
|
206
|
+
input_types=("single", "batch"),
|
|
207
|
+
source_levels=("third_party_extractor", "partial_extraction"),
|
|
208
|
+
requires_network=True,
|
|
209
|
+
)
|
|
210
|
+
return BackendDescriptor(
|
|
211
|
+
name=self.name,
|
|
212
|
+
layer="L3 browser/session fallback",
|
|
213
|
+
capabilities=(capability,),
|
|
214
|
+
description="Use OpenCLI browser extraction and return Markdown from stdout.",
|
|
215
|
+
provider="opencli.web",
|
|
216
|
+
is_fallback=True,
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
def health(self) -> BackendHealth:
|
|
220
|
+
if self._runner is not _run_command:
|
|
221
|
+
return BackendHealth(ok=True, dependencies={"opencli": "injected"})
|
|
222
|
+
if shutil.which("opencli") is None and shutil.which("opencli.cmd") is None:
|
|
223
|
+
return BackendHealth(
|
|
224
|
+
ok=False,
|
|
225
|
+
dependencies={"opencli": "missing"},
|
|
226
|
+
error_code="backend_dependency_missing",
|
|
227
|
+
error_summary="opencli executable is not available on PATH.",
|
|
228
|
+
)
|
|
229
|
+
return BackendHealth(ok=True, dependencies={"opencli": "available"})
|
|
230
|
+
|
|
231
|
+
def extract(self, request: BackendExtractRequest) -> TranscriptResult:
|
|
232
|
+
item = request.item
|
|
233
|
+
command = [
|
|
234
|
+
"opencli",
|
|
235
|
+
"web",
|
|
236
|
+
"read",
|
|
237
|
+
"--url",
|
|
238
|
+
item.url,
|
|
239
|
+
"--stdout",
|
|
240
|
+
"true",
|
|
241
|
+
"--download-images",
|
|
242
|
+
"false",
|
|
243
|
+
"--window",
|
|
244
|
+
"background",
|
|
245
|
+
"--keep-tab",
|
|
246
|
+
"false",
|
|
247
|
+
"-f",
|
|
248
|
+
"md",
|
|
249
|
+
]
|
|
250
|
+
exit_code, stdout, stderr = self._runner(command, self.timeout)
|
|
251
|
+
if exit_code != 0:
|
|
252
|
+
return _failed(self.name, f"backend_exit_{exit_code}", stderr or stdout)
|
|
253
|
+
|
|
254
|
+
text = stdout.strip()
|
|
255
|
+
if not text:
|
|
256
|
+
return _failed(self.name, "empty_result", "OpenCLI web read returned no Markdown text.")
|
|
257
|
+
|
|
258
|
+
return TranscriptResult(
|
|
259
|
+
text=text.rstrip() + "\n",
|
|
260
|
+
status="verified",
|
|
261
|
+
source_level="third_party_extractor",
|
|
262
|
+
backend=self.name,
|
|
263
|
+
title=_markdown_title(text) or item.title,
|
|
264
|
+
source_notes=[
|
|
265
|
+
"OpenCLI web read returned Markdown from browser/session fallback.",
|
|
266
|
+
"--download-images false was used; Openmd owns optional asset localization.",
|
|
267
|
+
"No raw stdout/stderr artifact was persisted by the provider.",
|
|
268
|
+
],
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _urllib_fetch(url: str, timeout: int = 30) -> str | bytes | None:
|
|
273
|
+
import urllib.request
|
|
274
|
+
|
|
275
|
+
request = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 (compatible; Openmd)", "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8"})
|
|
276
|
+
with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310 - http(s) only, checked by input validation
|
|
277
|
+
body = response.read(20 * 1024 * 1024)
|
|
278
|
+
charset = response.headers.get_content_charset()
|
|
279
|
+
if not body:
|
|
280
|
+
return None
|
|
281
|
+
# Without a declared charset, hand bytes to trafilatura (it sniffs <meta charset>, e.g. GBK).
|
|
282
|
+
return body.decode(charset, errors="replace") if charset else body
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _run_command(command: list[str], timeout: int) -> tuple[int, str, str]:
|
|
286
|
+
resolved = shutil.which(command[0]) or command[0]
|
|
287
|
+
try:
|
|
288
|
+
completed = run_interruptibly(
|
|
289
|
+
[resolved, *command[1:]],
|
|
290
|
+
capture_output=True,
|
|
291
|
+
text=True,
|
|
292
|
+
timeout=timeout,
|
|
293
|
+
check=False,
|
|
294
|
+
encoding="utf-8",
|
|
295
|
+
errors="replace",
|
|
296
|
+
)
|
|
297
|
+
except FileNotFoundError as exc:
|
|
298
|
+
return 127, "", sanitize_error_summary(exc) or "command not found"
|
|
299
|
+
except subprocess.TimeoutExpired as exc:
|
|
300
|
+
return (
|
|
301
|
+
124,
|
|
302
|
+
exc.stdout if isinstance(exc.stdout, str) else "",
|
|
303
|
+
f"command timed out after {timeout}s",
|
|
304
|
+
)
|
|
305
|
+
return completed.returncode, completed.stdout or "", completed.stderr or ""
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _metadata_dict(
|
|
309
|
+
metadata_extractor: TrafilaturaMetadataExtractor | None,
|
|
310
|
+
downloaded: object,
|
|
311
|
+
) -> dict[str, Any]:
|
|
312
|
+
if metadata_extractor is None:
|
|
313
|
+
return {}
|
|
314
|
+
try:
|
|
315
|
+
metadata = metadata_extractor(downloaded)
|
|
316
|
+
except Exception:
|
|
317
|
+
return {}
|
|
318
|
+
if metadata is None:
|
|
319
|
+
return {}
|
|
320
|
+
if isinstance(metadata, dict):
|
|
321
|
+
return dict(metadata)
|
|
322
|
+
as_dict = getattr(metadata, "as_dict", None)
|
|
323
|
+
if callable(as_dict):
|
|
324
|
+
try:
|
|
325
|
+
value = as_dict()
|
|
326
|
+
except Exception:
|
|
327
|
+
value = None
|
|
328
|
+
if isinstance(value, dict):
|
|
329
|
+
return value
|
|
330
|
+
result: dict[str, Any] = {}
|
|
331
|
+
for key in ("title", "author", "date", "published", "published_at"):
|
|
332
|
+
value = getattr(metadata, key, None)
|
|
333
|
+
if value:
|
|
334
|
+
result[key] = value
|
|
335
|
+
return result
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _metadata_value(metadata: dict[str, Any], *keys: str) -> str | None:
|
|
339
|
+
for key in keys:
|
|
340
|
+
value = metadata.get(key)
|
|
341
|
+
if value is not None and str(value).strip():
|
|
342
|
+
return str(value).strip()
|
|
343
|
+
return None
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def _markdown_title(markdown: str) -> str | None:
|
|
347
|
+
for line in markdown.splitlines():
|
|
348
|
+
stripped = line.strip()
|
|
349
|
+
if stripped.startswith("# "):
|
|
350
|
+
return stripped.removeprefix("# ").strip() or None
|
|
351
|
+
return None
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def _failed(backend: str, code: str, error: object) -> TranscriptResult:
|
|
355
|
+
return TranscriptResult(
|
|
356
|
+
text="",
|
|
357
|
+
status="failed",
|
|
358
|
+
source_level="partial_extraction",
|
|
359
|
+
backend=backend,
|
|
360
|
+
source_notes=["web article provider returned structured failure"],
|
|
361
|
+
error_code=code,
|
|
362
|
+
error_summary=sanitize_error_summary(error),
|
|
363
|
+
)
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""Runtime registration for WeChat group discovery providers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from openmd.adapters.wechat_group.wx_cli import (
|
|
6
|
+
OPENCLI_WX_HISTORY_DISCOVERY_BACKEND,
|
|
7
|
+
OpenCliWechatGroupHistoryProvider,
|
|
8
|
+
)
|
|
9
|
+
from openmd.backends.executor import BackendProviderRegistry, DiscoveryProviderRegistry
|
|
10
|
+
from openmd.backends.registry import BackendRuntimeOptions
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def register_backend_providers(
|
|
14
|
+
registry: BackendProviderRegistry,
|
|
15
|
+
options: BackendRuntimeOptions,
|
|
16
|
+
) -> None:
|
|
17
|
+
"""WeChat group extraction is intentionally not registered in mock/gated V1."""
|
|
18
|
+
return None
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def register_discovery_providers(
|
|
22
|
+
registry: DiscoveryProviderRegistry,
|
|
23
|
+
options: BackendRuntimeOptions,
|
|
24
|
+
) -> None:
|
|
25
|
+
"""Register OpenCLI WeChat group history only behind explicit browser gate."""
|
|
26
|
+
if not options.enable_browser_backends:
|
|
27
|
+
return
|
|
28
|
+
requested = options.requested_discovery_provider
|
|
29
|
+
if requested in {None, OPENCLI_WX_HISTORY_DISCOVERY_BACKEND}:
|
|
30
|
+
registry.register(
|
|
31
|
+
OPENCLI_WX_HISTORY_DISCOVERY_BACKEND,
|
|
32
|
+
OpenCliWechatGroupHistoryProvider(),
|
|
33
|
+
platforms=("wechat_group",),
|
|
34
|
+
default_for_platforms=requested is None,
|
|
35
|
+
)
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""Runtime registration for WeChat Moments discovery providers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from openmd.adapters.wechat_moments.wx_cli import (
|
|
6
|
+
OPENCLI_WX_SNS_FEED_DISCOVERY_BACKEND,
|
|
7
|
+
OpenCliWechatMomentsProvider,
|
|
8
|
+
)
|
|
9
|
+
from openmd.backends.executor import BackendProviderRegistry, DiscoveryProviderRegistry
|
|
10
|
+
from openmd.backends.registry import BackendRuntimeOptions
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def register_backend_providers(
|
|
14
|
+
registry: BackendProviderRegistry,
|
|
15
|
+
options: BackendRuntimeOptions,
|
|
16
|
+
) -> None:
|
|
17
|
+
"""WeChat Moments extraction is intentionally not registered in mock/gated V1."""
|
|
18
|
+
return None
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def register_discovery_providers(
|
|
22
|
+
registry: DiscoveryProviderRegistry,
|
|
23
|
+
options: BackendRuntimeOptions,
|
|
24
|
+
) -> None:
|
|
25
|
+
"""Register OpenCLI Moments discovery only behind explicit browser gate."""
|
|
26
|
+
if not options.enable_browser_backends:
|
|
27
|
+
return
|
|
28
|
+
requested = options.requested_discovery_provider
|
|
29
|
+
if requested in {None, OPENCLI_WX_SNS_FEED_DISCOVERY_BACKEND}:
|
|
30
|
+
registry.register(
|
|
31
|
+
OPENCLI_WX_SNS_FEED_DISCOVERY_BACKEND,
|
|
32
|
+
OpenCliWechatMomentsProvider(),
|
|
33
|
+
platforms=("wechat_moments",),
|
|
34
|
+
default_for_platforms=requested is None,
|
|
35
|
+
)
|