openmd-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. openmd/__init__.py +3 -0
  2. openmd/__main__.py +5 -0
  3. openmd/adapters/__init__.py +1 -0
  4. openmd/adapters/base.py +95 -0
  5. openmd/adapters/bilibili/__init__.py +11 -0
  6. openmd/adapters/bilibili/adapter.py +104 -0
  7. openmd/adapters/bilibili/discovery.py +33 -0
  8. openmd/adapters/douyin/__init__.py +11 -0
  9. openmd/adapters/douyin/adapter.py +117 -0
  10. openmd/adapters/douyin/discovery.py +36 -0
  11. openmd/adapters/douyin/sidecar.py +77 -0
  12. openmd/adapters/generic_media/__init__.py +14 -0
  13. openmd/adapters/high_privacy.py +77 -0
  14. openmd/adapters/local_audio/__init__.py +14 -0
  15. openmd/adapters/local_audio/adapter.py +169 -0
  16. openmd/adapters/local_audio/asr.py +58 -0
  17. openmd/adapters/local_audio/backends/__init__.py +1 -0
  18. openmd/adapters/local_audio/backends/auto.py +177 -0
  19. openmd/adapters/local_audio/backends/faster_whisper.py +285 -0
  20. openmd/adapters/local_audio/backends/funasr_sense_voice.py +14 -0
  21. openmd/adapters/local_audio/backends/qwen3_asr.py +254 -0
  22. openmd/adapters/opencli.py +770 -0
  23. openmd/adapters/pdf/__init__.py +11 -0
  24. openmd/adapters/pdf/adapter.py +153 -0
  25. openmd/adapters/pdf/backends/__init__.py +1 -0
  26. openmd/adapters/pdf/backends/markitdown.py +165 -0
  27. openmd/adapters/pdf/parser.py +56 -0
  28. openmd/adapters/qq_group/__init__.py +11 -0
  29. openmd/adapters/qq_group/adapter.py +289 -0
  30. openmd/adapters/qq_group/parsers/__init__.py +1 -0
  31. openmd/adapters/qq_group/parsers/html.py +60 -0
  32. openmd/adapters/qq_group/parsers/json.py +71 -0
  33. openmd/adapters/registry.py +149 -0
  34. openmd/adapters/screenshot/__init__.py +11 -0
  35. openmd/adapters/screenshot/adapter.py +154 -0
  36. openmd/adapters/screenshot/backends/__init__.py +1 -0
  37. openmd/adapters/screenshot/backends/paddle.py +15 -0
  38. openmd/adapters/screenshot/backends/rapidocr.py +176 -0
  39. openmd/adapters/screenshot/backends/tesseract.py +15 -0
  40. openmd/adapters/screenshot/ocr.py +84 -0
  41. openmd/adapters/wechat_group/__init__.py +11 -0
  42. openmd/adapters/wechat_group/adapter.py +160 -0
  43. openmd/adapters/wechat_group/wx_cli.py +268 -0
  44. openmd/adapters/wechat_moments/__init__.py +11 -0
  45. openmd/adapters/wechat_moments/adapter.py +167 -0
  46. openmd/adapters/wechat_moments/wx_cli.py +265 -0
  47. openmd/adapters/weixin/__init__.py +11 -0
  48. openmd/adapters/weixin/adapter.py +148 -0
  49. openmd/adapters/weixin/discovery.py +510 -0
  50. openmd/adapters/weixin/report.py +278 -0
  51. openmd/adapters/xiaohongshu.py +158 -0
  52. openmd/backends/__init__.py +1 -0
  53. openmd/backends/base.py +204 -0
  54. openmd/backends/bilibili.py +1480 -0
  55. openmd/backends/bilibili_wbi.py +277 -0
  56. openmd/backends/discourse.py +426 -0
  57. openmd/backends/douyin.py +868 -0
  58. openmd/backends/douyin_detail.py +124 -0
  59. openmd/backends/douyin_providers.py +602 -0
  60. openmd/backends/executor.py +546 -0
  61. openmd/backends/feishu.py +861 -0
  62. openmd/backends/generic_media.py +77 -0
  63. openmd/backends/interactive.py +60 -0
  64. openmd/backends/ocr.py +41 -0
  65. openmd/backends/opencli.py +82 -0
  66. openmd/backends/pdf.py +122 -0
  67. openmd/backends/registry.py +234 -0
  68. openmd/backends/screenshot.py +79 -0
  69. openmd/backends/video_metadata.py +167 -0
  70. openmd/backends/web_article.py +363 -0
  71. openmd/backends/wechat_group.py +35 -0
  72. openmd/backends/wechat_moments.py +35 -0
  73. openmd/backends/weixin.py +767 -0
  74. openmd/backends/x.py +455 -0
  75. openmd/backends/xiaoe.py +940 -0
  76. openmd/backends/xiaohongshu.py +911 -0
  77. openmd/backends/xiaohongshu_providers.py +876 -0
  78. openmd/backends/xiaoyuzhou.py +1017 -0
  79. openmd/backends/youtube.py +280 -0
  80. openmd/backends/youtube_discovery.py +1245 -0
  81. openmd/backends/youtube_transcript.py +213 -0
  82. openmd/browser_extension/background.js +231 -0
  83. openmd/browser_extension/icons/icon-128.png +0 -0
  84. openmd/browser_extension/icons/icon-16.png +0 -0
  85. openmd/browser_extension/icons/icon-32.png +0 -0
  86. openmd/browser_extension/icons/icon-48.png +0 -0
  87. openmd/browser_extension/icons/icon-512.png +0 -0
  88. openmd/browser_extension/idle.html +6 -0
  89. openmd/browser_extension/manifest.json +43 -0
  90. openmd/browser_extension/options.html +42 -0
  91. openmd/browser_extension/options.js +117 -0
  92. openmd/browser_extension/popup.css +71 -0
  93. openmd/browser_extension/popup.html +17 -0
  94. openmd/browser_extension/popup.js +59 -0
  95. openmd/browser_extension/vendor/opencli-bridge/LICENSE +190 -0
  96. openmd/browser_extension/vendor/opencli-bridge/VENDOR.md +31 -0
  97. openmd/browser_extension/vendor/opencli-bridge/background.js +2742 -0
  98. openmd/browser_host.py +503 -0
  99. openmd/browser_setup.py +106 -0
  100. openmd/cli.py +2012 -0
  101. openmd/console.py +32 -0
  102. openmd/core/__init__.py +1 -0
  103. openmd/core/capabilities.py +162 -0
  104. openmd/core/content_validation.py +323 -0
  105. openmd/core/coverage.py +163 -0
  106. openmd/core/detector.py +247 -0
  107. openmd/core/doctor.py +104 -0
  108. openmd/core/execution_policy.py +164 -0
  109. openmd/core/filename.py +64 -0
  110. openmd/core/input_validation.py +91 -0
  111. openmd/core/local_config.py +96 -0
  112. openmd/core/manifest.py +156 -0
  113. openmd/core/models.py +168 -0
  114. openmd/core/opencli_output.py +34 -0
  115. openmd/core/operation_contract.py +60 -0
  116. openmd/core/output.py +194 -0
  117. openmd/core/package_contract.py +335 -0
  118. openmd/core/package_validator.py +489 -0
  119. openmd/core/package_writer.py +666 -0
  120. openmd/core/planner.py +345 -0
  121. openmd/core/preferences.py +61 -0
  122. openmd/core/readable.py +129 -0
  123. openmd/core/receipt.py +285 -0
  124. openmd/core/renderer.py +185 -0
  125. openmd/core/runlog.py +208 -0
  126. openmd/core/secrets.py +313 -0
  127. openmd/doctor_view.py +68 -0
  128. openmd/interactive.py +453 -0
  129. openmd/ocr/__init__.py +1 -0
  130. openmd/ocr/breakers.py +84 -0
  131. openmd/ocr/config.py +54 -0
  132. openmd/ocr/documents.py +522 -0
  133. openmd/ocr/endpoint.py +72 -0
  134. openmd/ocr/layout.py +92 -0
  135. openmd/ocr/layout_filter.py +118 -0
  136. openmd/ocr/layout_worker.py +89 -0
  137. openmd/ocr/native_math.py +220 -0
  138. openmd/ocr/office.py +305 -0
  139. openmd/ocr/ollama.py +200 -0
  140. openmd/ocr/otsl.py +51 -0
  141. openmd/ocr/ovis-prompt.txt +1 -0
  142. openmd/ocr/pipeline.py +384 -0
  143. openmd/ocr/settings.json +25 -0
  144. openmd/ocr/tiling.py +631 -0
  145. openmd/post_video.py +154 -0
  146. openmd/probe.py +397 -0
  147. openmd/routing/__init__.py +1 -0
  148. openmd/routing/cards/builtin.json +620 -0
  149. openmd/routing/cards/defuddle.json +70 -0
  150. openmd/routing/cards/x-tweet-fetcher.json +71 -0
  151. openmd/routing/cards/xyz-dl.json +60 -0
  152. openmd/routing/cards/yutto.json +68 -0
  153. openmd/routing/entry_shim.py +71 -0
  154. openmd/routing/lifecycle.py +459 -0
  155. openmd/routing/loader.py +138 -0
  156. openmd/routing/providers.py +158 -0
  157. openmd/routing/readers.py +108 -0
  158. openmd/routing/routes.json +504 -0
  159. openmd/routing/runner.py +192 -0
  160. openmd/routing/view.py +40 -0
  161. openmd/runtime/__init__.py +1 -0
  162. openmd/runtime/_windows_job.py +234 -0
  163. openmd/runtime/asr.py +217 -0
  164. openmd/runtime/asr_worker.py +119 -0
  165. openmd/runtime/gpu_lock.py +81 -0
  166. openmd/runtime/gpu_quarantine.py +85 -0
  167. openmd/runtime/interpreters.py +407 -0
  168. openmd/runtime/managed_tools.py +204 -0
  169. openmd/runtime/models.py +202 -0
  170. openmd/runtime/services.py +175 -0
  171. openmd/runtime/tempdirs.py +227 -0
  172. openmd_cli-0.1.0.dist-info/METADATA +168 -0
  173. openmd_cli-0.1.0.dist-info/RECORD +177 -0
  174. openmd_cli-0.1.0.dist-info/WHEEL +5 -0
  175. openmd_cli-0.1.0.dist-info/entry_points.txt +2 -0
  176. openmd_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
  177. openmd_cli-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,79 @@
1
+ """Runtime providers for long screenshot OCR inputs."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from openmd.adapters.screenshot.adapter import ScreenshotOcrAdapter
6
+ from openmd.adapters.screenshot.backends.rapidocr import RapidOCRBackend
7
+ from openmd.adapters.screenshot.ocr import MissingOCRBackend, MockOCRBackend
8
+ from openmd.backends.base import (
9
+ BackendCapability,
10
+ BackendDescriptor,
11
+ BackendExtractRequest,
12
+ BackendHealth,
13
+ )
14
+ from openmd.backends.executor import BackendProviderRegistry
15
+ from openmd.backends.registry import BackendRuntimeOptions
16
+ from openmd.core.models import TranscriptResult
17
+
18
+
19
+ class ScreenshotOCRProvider:
20
+ def __init__(self, backend) -> None:
21
+ self.backend = backend
22
+ self.name = backend.name
23
+
24
+ def descriptor(self) -> BackendDescriptor:
25
+ source_level = "ocr" if isinstance(self.backend, MockOCRBackend) else "partial_extraction"
26
+ return BackendDescriptor(
27
+ name=self.name,
28
+ layer="local",
29
+ capabilities=(
30
+ BackendCapability(
31
+ platforms=("screenshot",),
32
+ source_types=("image_text",),
33
+ input_types=("single", "batch", "collection", "generic"),
34
+ source_levels=(source_level,),
35
+ requires_network=False,
36
+ requires_credentials=False,
37
+ ),
38
+ ),
39
+ description="Long screenshot OCR provider.",
40
+ provider="openmd.screenshot",
41
+ )
42
+
43
+ def health(self) -> BackendHealth:
44
+ is_available = getattr(self.backend, "is_available", None)
45
+ if callable(is_available) and not is_available():
46
+ return BackendHealth(
47
+ ok=False,
48
+ error_code="ocr_backend_missing",
49
+ error_summary=f"{self.name} is not installed or enabled.",
50
+ )
51
+ if isinstance(self.backend, MissingOCRBackend):
52
+ return BackendHealth(
53
+ ok=False,
54
+ error_code="ocr_backend_missing",
55
+ error_summary=f"{self.name} is not installed or enabled.",
56
+ )
57
+ return BackendHealth(ok=True, dependencies={self.name: "ok"})
58
+
59
+ def extract(self, request: BackendExtractRequest) -> TranscriptResult:
60
+ adapter = ScreenshotOcrAdapter("screenshot", request.primary_backend)
61
+ return adapter.extract_text(
62
+ request.item,
63
+ backend=request.primary_backend,
64
+ backend_chain=(request.primary_backend,),
65
+ allow_fallback=False,
66
+ )
67
+
68
+
69
+ def register_backend_providers(
70
+ registry: BackendProviderRegistry,
71
+ options: BackendRuntimeOptions,
72
+ ) -> None:
73
+ from openmd.backends.ocr import register_ocr_providers
74
+
75
+ register_ocr_providers(registry, options)
76
+ registry.register("screenshot.mock", ScreenshotOCRProvider(MockOCRBackend()))
77
+ registry.register("ocr.rapidocr", ScreenshotOCRProvider(RapidOCRBackend()))
78
+ registry.register("ocr.paddle", ScreenshotOCRProvider(MissingOCRBackend("ocr.paddle")))
79
+ registry.register("ocr.tesseract", ScreenshotOCRProvider(MissingOCRBackend("ocr.tesseract")))
@@ -0,0 +1,167 @@
1
+ """Optional public video metadata, separate from transcript acquisition.
2
+
3
+ Only the explicit public-network gate permits a request. Browser permission
4
+ alone never enables these requests, and metadata failure cannot lose text.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ import re
11
+ from collections.abc import Callable
12
+ from dataclasses import replace
13
+ from datetime import datetime, timezone
14
+ from typing import Any
15
+ from urllib.parse import urlencode
16
+ from urllib.request import Request, urlopen
17
+
18
+ from openmd.core.filename import short_hash
19
+ from openmd.core.models import ContentItem, TranscriptResult
20
+
21
+ from .youtube_transcript import extract_youtube_video_id
22
+
23
+
24
+ MetadataFetcher = Callable[[str, int], Any]
25
+ _YOUTUBE_ID = re.compile(r"^[A-Za-z0-9_-]{11}$")
26
+ _BILIBILI_ID = re.compile(r"BV[A-Za-z0-9]{10}")
27
+ _PLACEHOLDER_TITLES = frozenset({"untitled", "unknown", "video", "audio"})
28
+ _MAX_RESPONSE_BYTES = 512 * 1024
29
+
30
+
31
+ def _value(value: object, *, limit: int = 512) -> str | None:
32
+ if not isinstance(value, str):
33
+ return None
34
+ cleaned = " ".join(value.split()).strip()
35
+ return cleaned[:limit] if cleaned else None
36
+
37
+
38
+ def _identity(item: ContentItem, result: TranscriptResult) -> tuple[str, str | None]:
39
+ if item.platform == "youtube":
40
+ for candidate in (item.id, result.metadata.get("video_id"), extract_youtube_video_id(item.url)):
41
+ if isinstance(candidate, str) and _YOUTUBE_ID.fullmatch(candidate):
42
+ return candidate, candidate
43
+ else:
44
+ for candidate in (item.id, result.metadata.get("bvid"), item.url):
45
+ match = _BILIBILI_ID.search(candidate) if isinstance(candidate, str) else None
46
+ if match:
47
+ return match.group(), match.group()
48
+ return short_hash(item.url), None
49
+
50
+
51
+ def _source_title(value: object, *, identifier: str, platform: str) -> str | None:
52
+ title = _value(value)
53
+ if title is None or title.casefold() in _PLACEHOLDER_TITLES:
54
+ return None
55
+ if title.casefold() in {identifier.casefold(), f"{platform} · {identifier}".casefold()}:
56
+ return None
57
+ return title
58
+
59
+
60
+ def _fetch_public_json(url: str, timeout: int) -> Any:
61
+ # These fixed public endpoints do not receive session state or credentials.
62
+ request = Request(url, headers={"User-Agent": "Openmd/0.0", "Accept": "application/json"})
63
+ with urlopen(request, timeout=timeout) as response:
64
+ if not 200 <= int(getattr(response, "status", 200)) < 300:
65
+ raise ValueError("metadata_http_error")
66
+ raw = response.read(_MAX_RESPONSE_BYTES + 1)
67
+ if len(raw) > _MAX_RESPONSE_BYTES:
68
+ raise ValueError("metadata_response_too_large")
69
+ return json.loads(raw.decode("utf-8-sig"))
70
+
71
+
72
+ def _public_metadata(platform: str, identifier: str, fetch: MetadataFetcher, timeout: int) -> dict[str, str | None]:
73
+ if platform == "youtube":
74
+ url = "https://www.youtube.com/oembed?" + urlencode({
75
+ "url": f"https://www.youtube.com/watch?v={identifier}", "format": "json",
76
+ })
77
+ payload = fetch(url, timeout)
78
+ if not isinstance(payload, dict):
79
+ raise ValueError("metadata_payload_invalid")
80
+ return {
81
+ "title": _value(payload.get("title")),
82
+ "author": _value(payload.get("author_name"), limit=256),
83
+ "published_at": None, # oEmbed does not establish a publication date.
84
+ }
85
+
86
+ payload = fetch("https://api.bilibili.com/x/web-interface/view?" + urlencode({"bvid": identifier}), timeout)
87
+ if not isinstance(payload, dict) or payload.get("code") not in (0, "0"):
88
+ raise ValueError("metadata_payload_invalid")
89
+ data = payload.get("data")
90
+ if not isinstance(data, dict):
91
+ raise ValueError("metadata_payload_invalid")
92
+ owner = data.get("owner")
93
+ pubdate = data.get("pubdate")
94
+ published_at = None
95
+ if isinstance(pubdate, (int, float)) and not isinstance(pubdate, bool) and pubdate > 0:
96
+ try:
97
+ published_at = datetime.fromtimestamp(pubdate, timezone.utc).date().isoformat()
98
+ except (OverflowError, OSError, ValueError):
99
+ pass
100
+ return {
101
+ "title": _value(data.get("title")),
102
+ "author": _value(owner.get("name"), limit=256) if isinstance(owner, dict) else None,
103
+ "published_at": published_at,
104
+ }
105
+
106
+
107
+ def enrich_video_result(
108
+ item: ContentItem,
109
+ result: TranscriptResult,
110
+ *,
111
+ network_enabled: bool = False,
112
+ fetch_json: MetadataFetcher | None = None,
113
+ timeout: int = 8,
114
+ ) -> TranscriptResult:
115
+ """Fill absent video title/author, preserving all extraction evidence.
116
+
117
+ At most one bounded public request is made per call, only for a usable
118
+ YouTube/Bilibili result missing title or author and with network enabled.
119
+ Existing source metadata takes priority. No video title is inferred from
120
+ transcript words. ``fetch_json`` is an injectable JSON transport for local
121
+ verification; it obeys the same gate as the default transport.
122
+ """
123
+ if item.platform not in {"youtube", "bilibili"} or not result.has_usable_text or result.backend == "mock":
124
+ return result
125
+ identifier, fetch_identifier = _identity(item, result)
126
+ title = _source_title(result.title, identifier=identifier, platform=item.platform) or _source_title(
127
+ item.title, identifier=identifier, platform=item.platform,
128
+ )
129
+ author = _value(result.author, limit=256) or _value(item.author, limit=256)
130
+ published_at = result.published_at or item.published_at
131
+ status = "already_present" if title and author else "not_requested"
132
+ source = None
133
+ if not (title and author) and network_enabled and fetch_identifier:
134
+ try:
135
+ public = _public_metadata(item.platform, fetch_identifier, fetch_json or _fetch_public_json, timeout)
136
+ title = title or _source_title(public["title"], identifier=identifier, platform=item.platform)
137
+ author = author or public["author"]
138
+ published_at = published_at or public["published_at"]
139
+ status = "available" if title and author else "partial"
140
+ source = "youtube.oembed" if item.platform == "youtube" else "bilibili.public_view"
141
+ except Exception:
142
+ # A good transcript remains a good transcript even when metadata
143
+ # is blocked, times out, or has an unexpected shape. Never retain
144
+ # upstream payloads, URLs, headers or exception text in this note.
145
+ status = "unavailable"
146
+ missing = [field for field, value in (("title", title), ("author", author)) if not value]
147
+ notes = list(result.source_notes)
148
+ if not title:
149
+ label = "YouTube" if item.platform == "youtube" else "Bilibili"
150
+ title = f"{label} · {identifier}"
151
+ note = "视频标题未取得;保留已提取文字,以平台和视频 ID 命名。"
152
+ if note not in notes:
153
+ notes.append(note)
154
+ elif not author:
155
+ note = "视频作者未取得;保留已提取文字。"
156
+ if note not in notes:
157
+ notes.append(note)
158
+ return replace(
159
+ result,
160
+ title=title,
161
+ author=author,
162
+ published_at=published_at,
163
+ source_notes=notes,
164
+ metadata={**result.metadata, "video_metadata": {
165
+ "status": status, "source": source, "missing_fields": missing,
166
+ }},
167
+ )
@@ -0,0 +1,363 @@
1
+ """Generic web article extraction providers.
2
+
3
+ The primary provider wraps ``trafilatura`` as an optional local library. It is
4
+ registered only after ``--enable-network-backends`` because it fetches remote
5
+ pages. Browser/session fallbacks such as OpenCLI stay behind
6
+ ``--enable-browser-backends``.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import shutil
12
+ import subprocess
13
+ from collections.abc import Callable
14
+ from typing import Any
15
+
16
+ from openmd.backends.base import (
17
+ BackendCapability,
18
+ BackendDescriptor,
19
+ BackendExtractRequest,
20
+ BackendHealth,
21
+ )
22
+ from openmd.backends.executor import BackendProviderRegistry
23
+ from openmd.backends.registry import BackendRuntimeOptions
24
+ from openmd.core.models import TranscriptResult
25
+ from openmd.core.secrets import sanitize_error_summary
26
+ from openmd.runtime.interpreters import run_interruptibly
27
+
28
+
29
+ WEB_ARTICLE_TRAFILATURA_BACKEND = "web_article.trafilatura"
30
+ OPENCLI_WEB_READ_BACKEND = "opencli.web.read"
31
+ CRAWL4AI_WEB_ARTICLE_BACKEND = "crawl4ai.web_article"
32
+ DEFAULT_MIN_ARTICLE_CHARS = 40
33
+
34
+ CommandRunner = Callable[[list[str], int], tuple[int, str, str]]
35
+ TrafilaturaFetcher = Callable[[str], object]
36
+ TrafilaturaExtractor = Callable[..., str | None]
37
+ TrafilaturaMetadataExtractor = Callable[[object], object]
38
+
39
+
40
+ def register_backend_providers(
41
+ registry: BackendProviderRegistry,
42
+ options: BackendRuntimeOptions,
43
+ ) -> None:
44
+ """Register web article providers behind explicit runtime gates."""
45
+ requested = options.backend_override
46
+ from openmd.backends.discourse import DISCOURSE_BACKEND, DiscourseProvider
47
+
48
+ if (options.enable_network_backends or options.enable_browser_backends) and requested in {None, DISCOURSE_BACKEND}:
49
+ registry.register(DISCOURSE_BACKEND, DiscourseProvider(options))
50
+ if options.enable_network_backends and requested in {None, WEB_ARTICLE_TRAFILATURA_BACKEND}:
51
+ registry.register(WEB_ARTICLE_TRAFILATURA_BACKEND, WebArticleTrafilaturaProvider())
52
+
53
+ if options.enable_browser_backends and requested in {None, OPENCLI_WEB_READ_BACKEND}:
54
+ registry.register(OPENCLI_WEB_READ_BACKEND, OpenCliWebReadProvider())
55
+
56
+
57
+ class WebArticleTrafilaturaProvider:
58
+ """Extract generic static articles through the optional trafilatura API."""
59
+
60
+ name = WEB_ARTICLE_TRAFILATURA_BACKEND
61
+
62
+ def __init__(
63
+ self,
64
+ *,
65
+ fetcher: TrafilaturaFetcher | None = None,
66
+ extractor: TrafilaturaExtractor | None = None,
67
+ metadata_extractor: TrafilaturaMetadataExtractor | None = None,
68
+ min_article_chars: int = DEFAULT_MIN_ARTICLE_CHARS,
69
+ ) -> None:
70
+ self._fetcher = fetcher
71
+ self._extractor = extractor
72
+ self._metadata_extractor = metadata_extractor
73
+ self.min_article_chars = max(0, min_article_chars)
74
+
75
+ def descriptor(self) -> BackendDescriptor:
76
+ capability = BackendCapability(
77
+ platforms=("web_article",),
78
+ source_types=("article",),
79
+ input_types=("single", "batch"),
80
+ source_levels=("native_text", "partial_extraction"),
81
+ requires_network=True,
82
+ requires_credentials=False,
83
+ optional_config=("trafilatura",),
84
+ )
85
+ return BackendDescriptor(
86
+ name=self.name,
87
+ layer="L1 specialized extractor",
88
+ capabilities=(capability,),
89
+ description="Fetch and extract static web articles with optional local trafilatura.",
90
+ provider="openmd.backends.web_article",
91
+ )
92
+
93
+ def health(self) -> BackendHealth:
94
+ if self._fetcher is not None and self._extractor is not None:
95
+ return BackendHealth(ok=True, dependencies={"trafilatura": "injected"})
96
+ try:
97
+ import trafilatura # type: ignore[import-not-found] # noqa: F401
98
+ except ModuleNotFoundError:
99
+ return BackendHealth(
100
+ ok=False,
101
+ dependencies={"trafilatura": "missing"},
102
+ error_code="backend_dependency_missing",
103
+ error_summary="Install trafilatura or use a browser fallback backend.",
104
+ )
105
+ return BackendHealth(ok=True, dependencies={"trafilatura": "available"})
106
+
107
+ def extract(self, request: BackendExtractRequest) -> TranscriptResult:
108
+ item = request.item
109
+ if item.platform != "web_article":
110
+ return _failed(
111
+ self.name,
112
+ "unsupported_platform",
113
+ f"trafilatura web article provider only supports web_article, got {item.platform}.",
114
+ )
115
+
116
+ try:
117
+ fetcher, extractor, metadata_extractor = self._resolve_api()
118
+ except ModuleNotFoundError:
119
+ return _failed(
120
+ self.name,
121
+ "backend_dependency_missing",
122
+ "trafilatura is not installed; install it or enable browser fallback.",
123
+ )
124
+
125
+ try:
126
+ downloaded = fetcher(item.url)
127
+ except Exception as exc:
128
+ downloaded, fetch_error = None, exc
129
+ else:
130
+ fetch_error = None
131
+ if downloaded is None and self._fetcher is None:
132
+ # trafilatura's urllib3 client ignores system/SOCKS proxy settings;
133
+ # stdlib urllib honors HTTP(S)_PROXY and the Windows proxy.
134
+ try:
135
+ downloaded = _urllib_fetch(item.url)
136
+ except Exception as exc: # network errors are reported below
137
+ fetch_error = fetch_error or exc
138
+ if downloaded is None:
139
+ if fetch_error is not None:
140
+ return _failed(self.name, "web_article_fetch_failed", fetch_error)
141
+ return _failed(self.name, "web_article_fetch_empty", "trafilatura.fetch_url returned no document.")
142
+
143
+ try:
144
+ markdown = extractor(
145
+ downloaded,
146
+ output_format="markdown",
147
+ # Metadata is read separately below; True would embed a YAML
148
+ # block inside the body, duplicating the rendered front matter.
149
+ with_metadata=False,
150
+ include_images=True,
151
+ include_links=True,
152
+ url=item.url,
153
+ )
154
+ except Exception as exc:
155
+ return _failed(self.name, "web_article_extract_failed", exc)
156
+
157
+ text = (markdown or "").strip()
158
+ if len("".join(text.split())) < self.min_article_chars:
159
+ return _failed(
160
+ self.name,
161
+ "web_article_empty_content",
162
+ "trafilatura returned no usable article body.",
163
+ )
164
+
165
+ metadata = _metadata_dict(metadata_extractor, downloaded)
166
+ return TranscriptResult(
167
+ text=text.rstrip() + "\n",
168
+ status="verified",
169
+ source_level="native_text",
170
+ backend=self.name,
171
+ title=_metadata_value(metadata, "title"),
172
+ author=_metadata_value(metadata, "author"),
173
+ published_at=_metadata_value(metadata, "date", "published", "published_at"),
174
+ source_notes=[
175
+ "trafilatura extracted article Markdown locally from fetched HTML.",
176
+ "include_images=True and include_links=True were requested.",
177
+ "No browser, cloud API, media download, or ASR action was attempted.",
178
+ ],
179
+ metadata={"strategy": "trafilatura_api"},
180
+ )
181
+
182
+ def _resolve_api(
183
+ self,
184
+ ) -> tuple[TrafilaturaFetcher, TrafilaturaExtractor, TrafilaturaMetadataExtractor | None]:
185
+ if self._fetcher is not None and self._extractor is not None:
186
+ return self._fetcher, self._extractor, self._metadata_extractor
187
+
188
+ from trafilatura import extract, extract_metadata, fetch_url # type: ignore[import-not-found]
189
+
190
+ return fetch_url, extract, extract_metadata
191
+
192
+
193
+ class OpenCliWebReadProvider:
194
+ """Browser/session fallback for generic web pages through OpenCLI."""
195
+
196
+ name = OPENCLI_WEB_READ_BACKEND
197
+
198
+ def __init__(self, *, runner: CommandRunner | None = None, timeout: int = 180) -> None:
199
+ self._runner = runner or _run_command
200
+ self.timeout = timeout
201
+
202
+ def descriptor(self) -> BackendDescriptor:
203
+ capability = BackendCapability(
204
+ platforms=("web_article", "weixin"),
205
+ source_types=("article",),
206
+ input_types=("single", "batch"),
207
+ source_levels=("third_party_extractor", "partial_extraction"),
208
+ requires_network=True,
209
+ )
210
+ return BackendDescriptor(
211
+ name=self.name,
212
+ layer="L3 browser/session fallback",
213
+ capabilities=(capability,),
214
+ description="Use OpenCLI browser extraction and return Markdown from stdout.",
215
+ provider="opencli.web",
216
+ is_fallback=True,
217
+ )
218
+
219
+ def health(self) -> BackendHealth:
220
+ if self._runner is not _run_command:
221
+ return BackendHealth(ok=True, dependencies={"opencli": "injected"})
222
+ if shutil.which("opencli") is None and shutil.which("opencli.cmd") is None:
223
+ return BackendHealth(
224
+ ok=False,
225
+ dependencies={"opencli": "missing"},
226
+ error_code="backend_dependency_missing",
227
+ error_summary="opencli executable is not available on PATH.",
228
+ )
229
+ return BackendHealth(ok=True, dependencies={"opencli": "available"})
230
+
231
+ def extract(self, request: BackendExtractRequest) -> TranscriptResult:
232
+ item = request.item
233
+ command = [
234
+ "opencli",
235
+ "web",
236
+ "read",
237
+ "--url",
238
+ item.url,
239
+ "--stdout",
240
+ "true",
241
+ "--download-images",
242
+ "false",
243
+ "--window",
244
+ "background",
245
+ "--keep-tab",
246
+ "false",
247
+ "-f",
248
+ "md",
249
+ ]
250
+ exit_code, stdout, stderr = self._runner(command, self.timeout)
251
+ if exit_code != 0:
252
+ return _failed(self.name, f"backend_exit_{exit_code}", stderr or stdout)
253
+
254
+ text = stdout.strip()
255
+ if not text:
256
+ return _failed(self.name, "empty_result", "OpenCLI web read returned no Markdown text.")
257
+
258
+ return TranscriptResult(
259
+ text=text.rstrip() + "\n",
260
+ status="verified",
261
+ source_level="third_party_extractor",
262
+ backend=self.name,
263
+ title=_markdown_title(text) or item.title,
264
+ source_notes=[
265
+ "OpenCLI web read returned Markdown from browser/session fallback.",
266
+ "--download-images false was used; Openmd owns optional asset localization.",
267
+ "No raw stdout/stderr artifact was persisted by the provider.",
268
+ ],
269
+ )
270
+
271
+
272
+ def _urllib_fetch(url: str, timeout: int = 30) -> str | bytes | None:
273
+ import urllib.request
274
+
275
+ request = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 (compatible; Openmd)", "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8"})
276
+ with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310 - http(s) only, checked by input validation
277
+ body = response.read(20 * 1024 * 1024)
278
+ charset = response.headers.get_content_charset()
279
+ if not body:
280
+ return None
281
+ # Without a declared charset, hand bytes to trafilatura (it sniffs <meta charset>, e.g. GBK).
282
+ return body.decode(charset, errors="replace") if charset else body
283
+
284
+
285
+ def _run_command(command: list[str], timeout: int) -> tuple[int, str, str]:
286
+ resolved = shutil.which(command[0]) or command[0]
287
+ try:
288
+ completed = run_interruptibly(
289
+ [resolved, *command[1:]],
290
+ capture_output=True,
291
+ text=True,
292
+ timeout=timeout,
293
+ check=False,
294
+ encoding="utf-8",
295
+ errors="replace",
296
+ )
297
+ except FileNotFoundError as exc:
298
+ return 127, "", sanitize_error_summary(exc) or "command not found"
299
+ except subprocess.TimeoutExpired as exc:
300
+ return (
301
+ 124,
302
+ exc.stdout if isinstance(exc.stdout, str) else "",
303
+ f"command timed out after {timeout}s",
304
+ )
305
+ return completed.returncode, completed.stdout or "", completed.stderr or ""
306
+
307
+
308
+ def _metadata_dict(
309
+ metadata_extractor: TrafilaturaMetadataExtractor | None,
310
+ downloaded: object,
311
+ ) -> dict[str, Any]:
312
+ if metadata_extractor is None:
313
+ return {}
314
+ try:
315
+ metadata = metadata_extractor(downloaded)
316
+ except Exception:
317
+ return {}
318
+ if metadata is None:
319
+ return {}
320
+ if isinstance(metadata, dict):
321
+ return dict(metadata)
322
+ as_dict = getattr(metadata, "as_dict", None)
323
+ if callable(as_dict):
324
+ try:
325
+ value = as_dict()
326
+ except Exception:
327
+ value = None
328
+ if isinstance(value, dict):
329
+ return value
330
+ result: dict[str, Any] = {}
331
+ for key in ("title", "author", "date", "published", "published_at"):
332
+ value = getattr(metadata, key, None)
333
+ if value:
334
+ result[key] = value
335
+ return result
336
+
337
+
338
+ def _metadata_value(metadata: dict[str, Any], *keys: str) -> str | None:
339
+ for key in keys:
340
+ value = metadata.get(key)
341
+ if value is not None and str(value).strip():
342
+ return str(value).strip()
343
+ return None
344
+
345
+
346
+ def _markdown_title(markdown: str) -> str | None:
347
+ for line in markdown.splitlines():
348
+ stripped = line.strip()
349
+ if stripped.startswith("# "):
350
+ return stripped.removeprefix("# ").strip() or None
351
+ return None
352
+
353
+
354
+ def _failed(backend: str, code: str, error: object) -> TranscriptResult:
355
+ return TranscriptResult(
356
+ text="",
357
+ status="failed",
358
+ source_level="partial_extraction",
359
+ backend=backend,
360
+ source_notes=["web article provider returned structured failure"],
361
+ error_code=code,
362
+ error_summary=sanitize_error_summary(error),
363
+ )
@@ -0,0 +1,35 @@
1
+ """Runtime registration for WeChat group discovery providers."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from openmd.adapters.wechat_group.wx_cli import (
6
+ OPENCLI_WX_HISTORY_DISCOVERY_BACKEND,
7
+ OpenCliWechatGroupHistoryProvider,
8
+ )
9
+ from openmd.backends.executor import BackendProviderRegistry, DiscoveryProviderRegistry
10
+ from openmd.backends.registry import BackendRuntimeOptions
11
+
12
+
13
+ def register_backend_providers(
14
+ registry: BackendProviderRegistry,
15
+ options: BackendRuntimeOptions,
16
+ ) -> None:
17
+ """WeChat group extraction is intentionally not registered in mock/gated V1."""
18
+ return None
19
+
20
+
21
+ def register_discovery_providers(
22
+ registry: DiscoveryProviderRegistry,
23
+ options: BackendRuntimeOptions,
24
+ ) -> None:
25
+ """Register OpenCLI WeChat group history only behind explicit browser gate."""
26
+ if not options.enable_browser_backends:
27
+ return
28
+ requested = options.requested_discovery_provider
29
+ if requested in {None, OPENCLI_WX_HISTORY_DISCOVERY_BACKEND}:
30
+ registry.register(
31
+ OPENCLI_WX_HISTORY_DISCOVERY_BACKEND,
32
+ OpenCliWechatGroupHistoryProvider(),
33
+ platforms=("wechat_group",),
34
+ default_for_platforms=requested is None,
35
+ )
@@ -0,0 +1,35 @@
1
+ """Runtime registration for WeChat Moments discovery providers."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from openmd.adapters.wechat_moments.wx_cli import (
6
+ OPENCLI_WX_SNS_FEED_DISCOVERY_BACKEND,
7
+ OpenCliWechatMomentsProvider,
8
+ )
9
+ from openmd.backends.executor import BackendProviderRegistry, DiscoveryProviderRegistry
10
+ from openmd.backends.registry import BackendRuntimeOptions
11
+
12
+
13
+ def register_backend_providers(
14
+ registry: BackendProviderRegistry,
15
+ options: BackendRuntimeOptions,
16
+ ) -> None:
17
+ """WeChat Moments extraction is intentionally not registered in mock/gated V1."""
18
+ return None
19
+
20
+
21
+ def register_discovery_providers(
22
+ registry: DiscoveryProviderRegistry,
23
+ options: BackendRuntimeOptions,
24
+ ) -> None:
25
+ """Register OpenCLI Moments discovery only behind explicit browser gate."""
26
+ if not options.enable_browser_backends:
27
+ return
28
+ requested = options.requested_discovery_provider
29
+ if requested in {None, OPENCLI_WX_SNS_FEED_DISCOVERY_BACKEND}:
30
+ registry.register(
31
+ OPENCLI_WX_SNS_FEED_DISCOVERY_BACKEND,
32
+ OpenCliWechatMomentsProvider(),
33
+ platforms=("wechat_moments",),
34
+ default_for_platforms=requested is None,
35
+ )