openmd-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. openmd/__init__.py +3 -0
  2. openmd/__main__.py +5 -0
  3. openmd/adapters/__init__.py +1 -0
  4. openmd/adapters/base.py +95 -0
  5. openmd/adapters/bilibili/__init__.py +11 -0
  6. openmd/adapters/bilibili/adapter.py +104 -0
  7. openmd/adapters/bilibili/discovery.py +33 -0
  8. openmd/adapters/douyin/__init__.py +11 -0
  9. openmd/adapters/douyin/adapter.py +117 -0
  10. openmd/adapters/douyin/discovery.py +36 -0
  11. openmd/adapters/douyin/sidecar.py +77 -0
  12. openmd/adapters/generic_media/__init__.py +14 -0
  13. openmd/adapters/high_privacy.py +77 -0
  14. openmd/adapters/local_audio/__init__.py +14 -0
  15. openmd/adapters/local_audio/adapter.py +169 -0
  16. openmd/adapters/local_audio/asr.py +58 -0
  17. openmd/adapters/local_audio/backends/__init__.py +1 -0
  18. openmd/adapters/local_audio/backends/auto.py +177 -0
  19. openmd/adapters/local_audio/backends/faster_whisper.py +285 -0
  20. openmd/adapters/local_audio/backends/funasr_sense_voice.py +14 -0
  21. openmd/adapters/local_audio/backends/qwen3_asr.py +254 -0
  22. openmd/adapters/opencli.py +770 -0
  23. openmd/adapters/pdf/__init__.py +11 -0
  24. openmd/adapters/pdf/adapter.py +153 -0
  25. openmd/adapters/pdf/backends/__init__.py +1 -0
  26. openmd/adapters/pdf/backends/markitdown.py +165 -0
  27. openmd/adapters/pdf/parser.py +56 -0
  28. openmd/adapters/qq_group/__init__.py +11 -0
  29. openmd/adapters/qq_group/adapter.py +289 -0
  30. openmd/adapters/qq_group/parsers/__init__.py +1 -0
  31. openmd/adapters/qq_group/parsers/html.py +60 -0
  32. openmd/adapters/qq_group/parsers/json.py +71 -0
  33. openmd/adapters/registry.py +149 -0
  34. openmd/adapters/screenshot/__init__.py +11 -0
  35. openmd/adapters/screenshot/adapter.py +154 -0
  36. openmd/adapters/screenshot/backends/__init__.py +1 -0
  37. openmd/adapters/screenshot/backends/paddle.py +15 -0
  38. openmd/adapters/screenshot/backends/rapidocr.py +176 -0
  39. openmd/adapters/screenshot/backends/tesseract.py +15 -0
  40. openmd/adapters/screenshot/ocr.py +84 -0
  41. openmd/adapters/wechat_group/__init__.py +11 -0
  42. openmd/adapters/wechat_group/adapter.py +160 -0
  43. openmd/adapters/wechat_group/wx_cli.py +268 -0
  44. openmd/adapters/wechat_moments/__init__.py +11 -0
  45. openmd/adapters/wechat_moments/adapter.py +167 -0
  46. openmd/adapters/wechat_moments/wx_cli.py +265 -0
  47. openmd/adapters/weixin/__init__.py +11 -0
  48. openmd/adapters/weixin/adapter.py +148 -0
  49. openmd/adapters/weixin/discovery.py +510 -0
  50. openmd/adapters/weixin/report.py +278 -0
  51. openmd/adapters/xiaohongshu.py +158 -0
  52. openmd/backends/__init__.py +1 -0
  53. openmd/backends/base.py +204 -0
  54. openmd/backends/bilibili.py +1480 -0
  55. openmd/backends/bilibili_wbi.py +277 -0
  56. openmd/backends/discourse.py +426 -0
  57. openmd/backends/douyin.py +868 -0
  58. openmd/backends/douyin_detail.py +124 -0
  59. openmd/backends/douyin_providers.py +602 -0
  60. openmd/backends/executor.py +546 -0
  61. openmd/backends/feishu.py +861 -0
  62. openmd/backends/generic_media.py +77 -0
  63. openmd/backends/interactive.py +60 -0
  64. openmd/backends/ocr.py +41 -0
  65. openmd/backends/opencli.py +82 -0
  66. openmd/backends/pdf.py +122 -0
  67. openmd/backends/registry.py +234 -0
  68. openmd/backends/screenshot.py +79 -0
  69. openmd/backends/video_metadata.py +167 -0
  70. openmd/backends/web_article.py +363 -0
  71. openmd/backends/wechat_group.py +35 -0
  72. openmd/backends/wechat_moments.py +35 -0
  73. openmd/backends/weixin.py +767 -0
  74. openmd/backends/x.py +455 -0
  75. openmd/backends/xiaoe.py +940 -0
  76. openmd/backends/xiaohongshu.py +911 -0
  77. openmd/backends/xiaohongshu_providers.py +876 -0
  78. openmd/backends/xiaoyuzhou.py +1017 -0
  79. openmd/backends/youtube.py +280 -0
  80. openmd/backends/youtube_discovery.py +1245 -0
  81. openmd/backends/youtube_transcript.py +213 -0
  82. openmd/browser_extension/background.js +231 -0
  83. openmd/browser_extension/icons/icon-128.png +0 -0
  84. openmd/browser_extension/icons/icon-16.png +0 -0
  85. openmd/browser_extension/icons/icon-32.png +0 -0
  86. openmd/browser_extension/icons/icon-48.png +0 -0
  87. openmd/browser_extension/icons/icon-512.png +0 -0
  88. openmd/browser_extension/idle.html +6 -0
  89. openmd/browser_extension/manifest.json +43 -0
  90. openmd/browser_extension/options.html +42 -0
  91. openmd/browser_extension/options.js +117 -0
  92. openmd/browser_extension/popup.css +71 -0
  93. openmd/browser_extension/popup.html +17 -0
  94. openmd/browser_extension/popup.js +59 -0
  95. openmd/browser_extension/vendor/opencli-bridge/LICENSE +190 -0
  96. openmd/browser_extension/vendor/opencli-bridge/VENDOR.md +31 -0
  97. openmd/browser_extension/vendor/opencli-bridge/background.js +2742 -0
  98. openmd/browser_host.py +503 -0
  99. openmd/browser_setup.py +106 -0
  100. openmd/cli.py +2012 -0
  101. openmd/console.py +32 -0
  102. openmd/core/__init__.py +1 -0
  103. openmd/core/capabilities.py +162 -0
  104. openmd/core/content_validation.py +323 -0
  105. openmd/core/coverage.py +163 -0
  106. openmd/core/detector.py +247 -0
  107. openmd/core/doctor.py +104 -0
  108. openmd/core/execution_policy.py +164 -0
  109. openmd/core/filename.py +64 -0
  110. openmd/core/input_validation.py +91 -0
  111. openmd/core/local_config.py +96 -0
  112. openmd/core/manifest.py +156 -0
  113. openmd/core/models.py +168 -0
  114. openmd/core/opencli_output.py +34 -0
  115. openmd/core/operation_contract.py +60 -0
  116. openmd/core/output.py +194 -0
  117. openmd/core/package_contract.py +335 -0
  118. openmd/core/package_validator.py +489 -0
  119. openmd/core/package_writer.py +666 -0
  120. openmd/core/planner.py +345 -0
  121. openmd/core/preferences.py +61 -0
  122. openmd/core/readable.py +129 -0
  123. openmd/core/receipt.py +285 -0
  124. openmd/core/renderer.py +185 -0
  125. openmd/core/runlog.py +208 -0
  126. openmd/core/secrets.py +313 -0
  127. openmd/doctor_view.py +68 -0
  128. openmd/interactive.py +453 -0
  129. openmd/ocr/__init__.py +1 -0
  130. openmd/ocr/breakers.py +84 -0
  131. openmd/ocr/config.py +54 -0
  132. openmd/ocr/documents.py +522 -0
  133. openmd/ocr/endpoint.py +72 -0
  134. openmd/ocr/layout.py +92 -0
  135. openmd/ocr/layout_filter.py +118 -0
  136. openmd/ocr/layout_worker.py +89 -0
  137. openmd/ocr/native_math.py +220 -0
  138. openmd/ocr/office.py +305 -0
  139. openmd/ocr/ollama.py +200 -0
  140. openmd/ocr/otsl.py +51 -0
  141. openmd/ocr/ovis-prompt.txt +1 -0
  142. openmd/ocr/pipeline.py +384 -0
  143. openmd/ocr/settings.json +25 -0
  144. openmd/ocr/tiling.py +631 -0
  145. openmd/post_video.py +154 -0
  146. openmd/probe.py +397 -0
  147. openmd/routing/__init__.py +1 -0
  148. openmd/routing/cards/builtin.json +620 -0
  149. openmd/routing/cards/defuddle.json +70 -0
  150. openmd/routing/cards/x-tweet-fetcher.json +71 -0
  151. openmd/routing/cards/xyz-dl.json +60 -0
  152. openmd/routing/cards/yutto.json +68 -0
  153. openmd/routing/entry_shim.py +71 -0
  154. openmd/routing/lifecycle.py +459 -0
  155. openmd/routing/loader.py +138 -0
  156. openmd/routing/providers.py +158 -0
  157. openmd/routing/readers.py +108 -0
  158. openmd/routing/routes.json +504 -0
  159. openmd/routing/runner.py +192 -0
  160. openmd/routing/view.py +40 -0
  161. openmd/runtime/__init__.py +1 -0
  162. openmd/runtime/_windows_job.py +234 -0
  163. openmd/runtime/asr.py +217 -0
  164. openmd/runtime/asr_worker.py +119 -0
  165. openmd/runtime/gpu_lock.py +81 -0
  166. openmd/runtime/gpu_quarantine.py +85 -0
  167. openmd/runtime/interpreters.py +407 -0
  168. openmd/runtime/managed_tools.py +204 -0
  169. openmd/runtime/models.py +202 -0
  170. openmd/runtime/services.py +175 -0
  171. openmd/runtime/tempdirs.py +227 -0
  172. openmd_cli-0.1.0.dist-info/METADATA +168 -0
  173. openmd_cli-0.1.0.dist-info/RECORD +177 -0
  174. openmd_cli-0.1.0.dist-info/WHEEL +5 -0
  175. openmd_cli-0.1.0.dist-info/entry_points.txt +2 -0
  176. openmd_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
  177. openmd_cli-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,163 @@
1
+ """Small reusable counters for collection coverage summaries.
2
+
3
+ The helper intentionally keeps only bounded enums and integer counts. It does
4
+ not retain URLs, output paths, backend payloads, or other acquisition data, so
5
+ platform summaries can safely serialize the result without leaking transient
6
+ locators.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from collections import Counter, defaultdict
12
+ from dataclasses import dataclass, field
13
+ from typing import Any, Iterable, Mapping
14
+
15
+ from .models import ContentItem, ExtractionComponent, TranscriptResult
16
+ from .package_contract import package_source_level
17
+
18
+
19
+ _ITEM_STATUSES = ("verified", "partial", "failed")
20
+ _BLOCKING_COMPONENT_STATUSES = {"partial", "failed", "retryable", "auth_challenge"}
21
+ _CONTENT_TYPES = ("text", "image_text", "video", "mixed")
22
+
23
+
24
+ @dataclass
25
+ class CollectionCoverage:
26
+ """Accumulate item outcomes and component provenance without identifiers."""
27
+
28
+ attempted: int = 0
29
+ skipped_existing: int = 0
30
+ terminal_gap_items: int = 0
31
+ retryable_items: int = 0
32
+ _item_statuses: Counter[str] = field(default_factory=Counter)
33
+ _component_statuses: dict[str, Counter[str]] = field(
34
+ default_factory=lambda: defaultdict(Counter)
35
+ )
36
+ _component_sources: dict[str, Counter[str]] = field(
37
+ default_factory=lambda: defaultdict(Counter)
38
+ )
39
+ _source_levels: Counter[str] = field(default_factory=Counter)
40
+
41
+ def record_result(
42
+ self,
43
+ result: TranscriptResult,
44
+ *,
45
+ skipped_existing: bool = False,
46
+ force_failed: bool = False,
47
+ ) -> None:
48
+ self.record(
49
+ status="failed" if force_failed else result.status,
50
+ components=result.normalized_components,
51
+ skipped_existing=skipped_existing,
52
+ )
53
+
54
+ def record_metadata(
55
+ self,
56
+ metadata: Mapping[str, Any],
57
+ *,
58
+ skipped_existing: bool = False,
59
+ ) -> None:
60
+ raw_components = metadata.get("components")
61
+ components = raw_components if isinstance(raw_components, list) else []
62
+ self.record(
63
+ status=str(metadata.get("status") or "failed"),
64
+ components=components,
65
+ skipped_existing=skipped_existing,
66
+ )
67
+
68
+ def record_failure(self) -> None:
69
+ self.record(status="failed", components=[])
70
+
71
+ def record(
72
+ self,
73
+ *,
74
+ status: str,
75
+ components: Iterable[ExtractionComponent | Mapping[str, Any]],
76
+ skipped_existing: bool = False,
77
+ ) -> None:
78
+ normalized_status = status if status in _ITEM_STATUSES else "failed"
79
+ self.attempted += 1
80
+ self._item_statuses[normalized_status] += 1
81
+ if skipped_existing:
82
+ self.skipped_existing += 1
83
+
84
+ has_terminal_gap = False
85
+ has_blocking_component = normalized_status == "failed"
86
+ for component in components:
87
+ component_id, component_status, source_level = _component_projection(component)
88
+ if not component_id:
89
+ continue
90
+ source_level = package_source_level(source_level)
91
+ self._component_statuses[component_id][component_status] += 1
92
+ self._component_sources[component_id][source_level] += 1
93
+ self._source_levels[source_level] += 1
94
+ has_terminal_gap = has_terminal_gap or component_status == "unavailable"
95
+ has_blocking_component = (
96
+ has_blocking_component
97
+ or component_status in _BLOCKING_COMPONENT_STATUSES
98
+ )
99
+
100
+ if has_terminal_gap:
101
+ self.terminal_gap_items += 1
102
+ if has_blocking_component:
103
+ self.retryable_items += 1
104
+
105
+ def as_dict(self) -> dict[str, Any]:
106
+ component_counts: dict[str, dict[str, int]] = {}
107
+ component_source_counts: dict[str, dict[str, int]] = {}
108
+ for component_id in sorted(self._component_statuses):
109
+ statuses = self._component_statuses[component_id]
110
+ component_counts[component_id] = {
111
+ "total": sum(statuses.values()),
112
+ **{key: int(value) for key, value in sorted(statuses.items())},
113
+ }
114
+ component_source_counts[component_id] = {
115
+ key: int(value)
116
+ for key, value in sorted(self._component_sources[component_id].items())
117
+ }
118
+
119
+ return {
120
+ "attempted": self.attempted,
121
+ "verified": int(self._item_statuses["verified"]),
122
+ "partial": int(self._item_statuses["partial"]),
123
+ "failed": int(self._item_statuses["failed"]),
124
+ "skipped_existing": self.skipped_existing,
125
+ "terminal_gaps": sum(
126
+ counts["unavailable"] for counts in self._component_statuses.values()
127
+ ),
128
+ "terminal_gap_items": self.terminal_gap_items,
129
+ "retryable": sum(
130
+ counts["retryable"] for counts in self._component_statuses.values()
131
+ ),
132
+ "retryable_items": self.retryable_items,
133
+ "component_counts": component_counts,
134
+ "component_source_counts": component_source_counts,
135
+ "source_counts": {
136
+ key: int(value) for key, value in sorted(self._source_levels.items())
137
+ },
138
+ }
139
+
140
+
141
+ def collection_content_type_counts(items: Iterable[ContentItem]) -> dict[str, int]:
142
+ """Return stable text/image_text/video/mixed counts for discovered items."""
143
+ counts = Counter({key: 0 for key in _CONTENT_TYPES})
144
+ for item in items:
145
+ raw_kind = item.raw_meta.get("media_kind") if isinstance(item.raw_meta, dict) else None
146
+ kind = str(raw_kind or item.source_type or "mixed").strip().lower().replace("-", "_")
147
+ if kind in {"article", "document", "text_first", "text_only"}:
148
+ kind = "text"
149
+ if kind not in _CONTENT_TYPES:
150
+ kind = "mixed"
151
+ counts[kind] += 1
152
+ return {key: int(counts[key]) for key in _CONTENT_TYPES}
153
+
154
+
155
+ def _component_projection(
156
+ component: ExtractionComponent | Mapping[str, Any],
157
+ ) -> tuple[str, str, str]:
158
+ if isinstance(component, ExtractionComponent):
159
+ return component.id, component.status, component.source_level
160
+ component_id = str(component.get("id") or "").strip()
161
+ status = str(component.get("status") or "failed").strip()
162
+ source_level = str(component.get("source_level") or "partial_extraction").strip()
163
+ return component_id, status, source_level
@@ -0,0 +1,247 @@
1
+ """Conservative URL platform detector for Openmd V0 categories."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from pathlib import Path
7
+ from urllib.parse import unquote, urlparse
8
+
9
+
10
+ PLATFORM_IDS: tuple[str, ...] = (
11
+ "bilibili",
12
+ "youtube",
13
+ "weixin",
14
+ "wechat_group",
15
+ "wechat_moments",
16
+ "qq_group",
17
+ "xiaoyuzhou",
18
+ "douyin",
19
+ "tiktok",
20
+ "xiaohongshu",
21
+ "xiaoe",
22
+ "x",
23
+ "feishu",
24
+ "web_article",
25
+ "generic_media",
26
+ "pdf",
27
+ "screenshot",
28
+ )
29
+
30
+ MEDIA_EXTENSIONS: tuple[str, ...] = (
31
+ ".mp4",
32
+ ".m4v",
33
+ ".mov",
34
+ ".webm",
35
+ ".mkv",
36
+ ".mp3",
37
+ ".m4a",
38
+ ".wav",
39
+ ".flac",
40
+ ".m3u8",
41
+ )
42
+
43
+ # Keep the legacy `pdf` platform identity for the approved PDF/Office lane.
44
+ DOCUMENT_EXTENSIONS: tuple[str, ...] = (".pdf", ".docx", ".pptx", ".xlsx")
45
+
46
+ IMAGE_TEXT_EXTENSIONS: tuple[str, ...] = (
47
+ ".png",
48
+ ".jpg",
49
+ ".jpeg",
50
+ ".webp",
51
+ ".bmp",
52
+ )
53
+
54
+ QQ_EXPORT_EXTENSIONS: tuple[str, ...] = (".json", ".html", ".txt")
55
+
56
+ _BV_RE = re.compile(r"^BV[0-9A-Za-z]+$")
57
+ _URL_RE = re.compile(r"https?://[^\s,。]+")
58
+
59
+
60
+ def extract_first_url(input_value: str) -> str:
61
+ value = input_value.strip()
62
+ match = _URL_RE.search(value)
63
+ if not match:
64
+ return value
65
+ return match.group(0).rstrip(".,,。;;!!))]")
66
+
67
+
68
+ def _host_and_path(input_value: str) -> tuple[str, str]:
69
+ value = extract_first_url(input_value)
70
+ parse_value = value if "://" in value else f"https://{value}"
71
+ parsed = urlparse(parse_value)
72
+ return parsed.netloc.lower(), parsed.path.lower()
73
+
74
+
75
+ def _host_endswith(host: str, suffixes: tuple[str, ...]) -> bool:
76
+ return any(host == suffix or host.endswith(f".{suffix}") for suffix in suffixes)
77
+
78
+
79
+ def looks_like_media_url(input_value: str) -> bool:
80
+ _, path = _host_and_path(input_value)
81
+ return path.endswith(MEDIA_EXTENSIONS)
82
+
83
+
84
+ def _path_for_extension_check(input_value: str) -> str:
85
+ value = extract_first_url(input_value)
86
+ parsed = urlparse(value)
87
+ if parsed.scheme == "file":
88
+ return parsed.path.lower()
89
+ if parsed.scheme in {"http", "https"}:
90
+ return parsed.path.lower()
91
+ return value.lower()
92
+
93
+
94
+ def detect_platform(input_value: str, *, explicit_media: bool = False) -> str:
95
+ value = input_value.strip()
96
+ if explicit_media:
97
+ return "generic_media"
98
+ directory_platform = _local_directory_platform(value)
99
+ if directory_platform is not None:
100
+ return directory_platform
101
+ if _is_weixin_official_account_selector(value):
102
+ return "weixin"
103
+ if _is_wechat_moments_selector(value):
104
+ return "wechat_moments"
105
+ if _is_wechat_group_selector(value):
106
+ return "wechat_group"
107
+ if _is_qq_group_selector(value) or _looks_like_qq_export_path(value):
108
+ return "qq_group"
109
+ if _BV_RE.match(value):
110
+ return "bilibili"
111
+ if re.fullmatch(r"\d{15,20}", value):
112
+ return "douyin"
113
+
114
+ extension_path = _path_for_extension_check(value)
115
+ if extension_path.endswith(DOCUMENT_EXTENSIONS):
116
+ return "pdf"
117
+ if extension_path.endswith(IMAGE_TEXT_EXTENSIONS):
118
+ return "screenshot"
119
+ if extension_path.endswith(MEDIA_EXTENSIONS):
120
+ return "generic_media"
121
+
122
+ host, path = _host_and_path(value)
123
+ if _host_endswith(host, ("bilibili.com", "b23.tv")):
124
+ return "bilibili"
125
+ if _host_endswith(host, ("youtube.com", "youtu.be", "youtube-nocookie.com")):
126
+ return "youtube"
127
+ if host == "mp.weixin.qq.com":
128
+ return "weixin"
129
+ if _host_endswith(host, ("xiaoyuzhoufm.com",)):
130
+ return "xiaoyuzhou"
131
+ if _host_endswith(host, ("douyin.com", "iesdouyin.com")):
132
+ return "douyin"
133
+ if _host_endswith(host, ("tiktok.com",)):
134
+ return "tiktok"
135
+ if _host_endswith(host, ("xiaohongshu.com", "xhslink.com")):
136
+ return "xiaohongshu"
137
+ if _host_endswith(host, ("xet.citv.cn", "xet.pomoho.com", "xiaoeknow.com")) or "xiaoe" in host:
138
+ return "xiaoe"
139
+ if _host_endswith(host, ("x.com", "twitter.com")):
140
+ return "x"
141
+ if _host_endswith(host, ("feishu.cn", "larksuite.com")):
142
+ return "feishu"
143
+ if path.endswith(MEDIA_EXTENSIONS):
144
+ return "generic_media"
145
+ return "web_article"
146
+
147
+
148
+ def _local_directory_platform(value: str) -> str | None:
149
+ path = _local_path_for_detection(value)
150
+ if path is None or not path.is_dir():
151
+ return None
152
+
153
+ seen_pdf = False
154
+ seen_image = False
155
+ seen_media = False
156
+ checked = 0
157
+ try:
158
+ for candidate in path.rglob("*"):
159
+ if not candidate.is_file():
160
+ continue
161
+ suffix = candidate.suffix.lower()
162
+ seen_pdf = seen_pdf or suffix in DOCUMENT_EXTENSIONS
163
+ seen_image = seen_image or suffix in IMAGE_TEXT_EXTENSIONS
164
+ seen_media = seen_media or suffix in MEDIA_EXTENSIONS
165
+ checked += 1
166
+ if checked >= 200 or seen_pdf:
167
+ break
168
+ except OSError:
169
+ return None
170
+
171
+ if seen_pdf:
172
+ return "pdf"
173
+ if seen_image:
174
+ return "screenshot"
175
+ if seen_media:
176
+ return "generic_media"
177
+ return None
178
+
179
+
180
+ def _local_path_for_detection(value: str) -> Path | None:
181
+ parsed = urlparse(value)
182
+ if parsed.scheme in {"http", "https"}:
183
+ return None
184
+ if len(parsed.scheme) == 1 and len(value) >= 3 and value[1] == ":" and value[2] in {"\\", "/"}:
185
+ return Path(value)
186
+ if parsed.scheme == "file":
187
+ raw_path = unquote(parsed.path)
188
+ if parsed.netloc and not raw_path:
189
+ raw_path = parsed.netloc
190
+ if raw_path.startswith("/") and len(raw_path) >= 4 and raw_path[2] == ":":
191
+ raw_path = raw_path[1:]
192
+ return Path(raw_path)
193
+ if parsed.scheme:
194
+ return None
195
+ return Path(value)
196
+
197
+
198
+ def _is_wechat_group_selector(value: str) -> bool:
199
+ parsed = urlparse(value.strip())
200
+ if parsed.scheme in {"wx", "wechat"} and parsed.netloc.lower() == "group":
201
+ return bool(parsed.path.strip("/"))
202
+ if parsed.scheme in {"wx-group", "wechat-group"}:
203
+ return bool(parsed.netloc or parsed.path.strip("/"))
204
+ return False
205
+
206
+
207
+ def _is_weixin_official_account_selector(value: str) -> bool:
208
+ parsed = urlparse(value.strip())
209
+ if parsed.scheme in {"weixin", "wx-official", "wechat-official"}:
210
+ return bool(parsed.netloc or parsed.path.strip("/"))
211
+ if parsed.scheme == "wechat" and parsed.netloc.lower() in {
212
+ "official",
213
+ "official-account",
214
+ "oa",
215
+ "author",
216
+ "account",
217
+ "fakeid",
218
+ "biz",
219
+ }:
220
+ return bool(parsed.path.strip("/") or parsed.query.strip())
221
+ return False
222
+
223
+
224
+ def _is_wechat_moments_selector(value: str) -> bool:
225
+ parsed = urlparse(value.strip())
226
+ if parsed.scheme in {"wx", "wechat"} and parsed.netloc.lower() in {"moments", "sns"}:
227
+ return bool(parsed.path.strip("/"))
228
+ if parsed.scheme in {"wx-moments", "wechat-moments"}:
229
+ return bool(parsed.netloc or parsed.path.strip("/"))
230
+ return False
231
+
232
+
233
+ def _is_qq_group_selector(value: str) -> bool:
234
+ parsed = urlparse(value.strip())
235
+ if parsed.scheme == "qq" and parsed.netloc.lower() == "group":
236
+ return bool(parsed.path.strip("/"))
237
+ if parsed.scheme in {"qq-group", "qq_export", "qq-export"}:
238
+ return bool(parsed.netloc or parsed.path.strip("/"))
239
+ return False
240
+
241
+
242
+ def _looks_like_qq_export_path(value: str) -> bool:
243
+ extension_path = _path_for_extension_check(value)
244
+ if not extension_path.endswith(QQ_EXPORT_EXTENSIONS):
245
+ return False
246
+ filename = extension_path.replace("\\", "/").rsplit("/", 1)[-1]
247
+ return "qq" in filename or "group-chat" in filename
openmd/core/doctor.py ADDED
@@ -0,0 +1,104 @@
1
+ """Safe local-only doctor projections for Openmd capability profiles."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import importlib.util
6
+ import shutil
7
+ from typing import Any
8
+
9
+ from .capabilities import validate_capability_manifest
10
+
11
+
12
+ DOCTOR_PROFILES = ("all", "core", "web", "media", "documents", "authenticated-social", "tools")
13
+
14
+
15
+ def _module_available(name: str) -> bool:
16
+ try:
17
+ return importlib.util.find_spec(name) is not None
18
+ except (ImportError, ValueError):
19
+ return False
20
+
21
+
22
+ def _media_diagnostics() -> dict[str, Any]:
23
+ from openmd.runtime.interpreters import select_asr_runtime
24
+ from openmd.runtime.models import configured_models
25
+
26
+ runtime = select_asr_runtime()
27
+ ffmpeg_available = shutil.which("ffmpeg") is not None
28
+ ready = bool(runtime.probe.get("ready")) and ffmpeg_available
29
+ next_action = "Local CUDA ASR runtime and models are ready" if ready else runtime.reason
30
+ if not ffmpeg_available:
31
+ next_action = "Install ffmpeg and make it available on PATH, then rerun openmd doctor --profile media"
32
+ return {
33
+ "ffmpeg_on_path": ffmpeg_available,
34
+ "local_asr_dependency": _module_available("faster_whisper"),
35
+ "asr_runtime": runtime.as_dict(),
36
+ "models": {key: model.as_dict() for key, model in configured_models().items()},
37
+ "ready": ready,
38
+ "next_action": next_action,
39
+ }
40
+
41
+
42
+ def _document_diagnostics() -> dict[str, Any]:
43
+ from openmd.ocr.config import settings
44
+ from openmd.ocr.layout import probe_layout
45
+ from openmd.ocr.office import soffice_path
46
+ from openmd.ocr.ollama import probe_ollama
47
+ from openmd.runtime.gpu_lock import gpu_lock_path
48
+ from openmd.runtime.gpu_quarantine import quarantine_path
49
+
50
+ try:
51
+ ollama = probe_ollama(settings())
52
+ except ValueError as exc:
53
+ ollama = {"reachable": False, "ready": False, "models": {}, "error_code": getattr(exc, "code", "ocr_setting_invalid")}
54
+ layout = probe_layout()
55
+ layout.update({"device": "cpu", "cpu_only": True, "cuda_visible_policy": "reject", "gpu_lease_inherited": False})
56
+ dependencies = {"pymupdf_available": _module_available("pymupdf"), "pillow_available": _module_available("PIL"), "numpy_available": _module_available("numpy")}
57
+ ready = all(dependencies.values()) and ollama["ready"] and layout["ready"]
58
+ return {"markitdown_available": _module_available("markitdown"), "ocr_dependency_available": _module_available("rapidocr_onnxruntime"), **dependencies, "ollama": ollama, "layout_runtime": layout, "libreoffice_available": soffice_path() is not None, "gpu_lock": {"path": str(gpu_lock_path()), "implementation": "OS file lock shared with ASR", "crash_recovery": "OS releases lease; persistent file is reused", "quarantine_exists": quarantine_path().exists(), "quarantine_recovery": "read-only wait for empty /api/ps; never unload other clients", "marker_only_reset": "openmd doctor --clear-gpu-quarantine"}, "ready": ready, "network_access": "configured_ollama_metadata_only", "next_action": "Document OCR dependencies are ready" if ready else "Start Ollama yourself and provide existing models / configure ocr_python and model_roots; Openmd does not install, download, pull or start services."}
59
+
60
+
61
+ def build_doctor_report(profile: str) -> dict[str, Any]:
62
+ if profile not in DOCTOR_PROFILES:
63
+ raise ValueError(f"unsupported doctor profile: {profile}")
64
+ try:
65
+ validate_capability_manifest()
66
+ manifest_valid = True
67
+ except ValueError:
68
+ manifest_valid = False
69
+ diagnostics: dict[str, Any] = {
70
+ "profile": profile,
71
+ "manifest_valid": manifest_valid,
72
+ "network_access": "not_probed",
73
+ "private_session_state": "not_read",
74
+ "account_actions": "not_performed",
75
+ "ready": manifest_valid,
76
+ }
77
+ if profile == "all":
78
+ diagnostics.update(_media_diagnostics())
79
+ diagnostics["openmd_on_path"] = shutil.which("openmd") is not None
80
+ diagnostics["ready"] = bool(diagnostics["ready"] and manifest_valid)
81
+ diagnostics["optional_profiles"] = {
82
+ "web": {"opencli_on_path": shutil.which("opencli") is not None, "network_gate": "explicit_opt_in"},
83
+ "documents": _document_diagnostics(),
84
+ }
85
+ elif profile == "core":
86
+ diagnostics.update({"openmd_on_path": shutil.which("openmd") is not None})
87
+ elif profile == "web":
88
+ diagnostics.update({"opencli_on_path": shutil.which("opencli") is not None, "network_gate": "explicit_opt_in"})
89
+ elif profile == "media":
90
+ diagnostics.update(_media_diagnostics())
91
+ diagnostics["ready"] = bool(diagnostics["ready"] and manifest_valid)
92
+ elif profile == "documents":
93
+ diagnostics.update(_document_diagnostics())
94
+ diagnostics["ready"] = manifest_valid and diagnostics["ready"]
95
+ elif profile == "tools":
96
+ # Populated by the tool-card diagnostics without executing a command.
97
+ from openmd.routing.providers import doctor_tools
98
+
99
+ tool_report = doctor_tools()
100
+ diagnostics["tools"] = tool_report
101
+ diagnostics["ready"] = manifest_valid and bool(tool_report.get("ok", False))
102
+ else:
103
+ diagnostics.update({"session_owner": "browser_platform_or_sidecar", "private_session_probe": False})
104
+ return diagnostics
@@ -0,0 +1,164 @@
1
+ """Execution identity and compact package writer ownership rules."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import errno
6
+ import hashlib
7
+ import json
8
+ import os
9
+ import tempfile
10
+ import uuid
11
+ from dataclasses import dataclass, field
12
+ from pathlib import Path
13
+
14
+
15
+ FINGERPRINT_CONTRACT_VERSION = "openmd.low-token.v1"
16
+ COMPACT_LOCK_VERSION = "openmd.compact-lock.v2"
17
+
18
+
19
+ def execution_fingerprint(
20
+ *,
21
+ requested_profile: str | None,
22
+ requested_modalities: tuple[str, ...],
23
+ provider_chain: tuple[str, ...],
24
+ backend_override: str | None,
25
+ model: str | None,
26
+ language: str | None,
27
+ authorization_flags: dict[str, bool] | None = None,
28
+ temporary_url: str | None = None,
29
+ extraction_contract: dict[str, object] | None = None,
30
+ effective_config: dict[str, object] | None = None,
31
+ ) -> str:
32
+ """Return a stable fingerprint without credentials, gates, or transient locators."""
33
+ del authorization_flags, temporary_url
34
+ payload = {
35
+ "contract_version": FINGERPRINT_CONTRACT_VERSION,
36
+ "requested_profile": requested_profile,
37
+ "requested_modalities": list(requested_modalities),
38
+ "provider_chain": list(provider_chain),
39
+ "backend_override": backend_override,
40
+ "model": model,
41
+ "language": language,
42
+ "extraction_contract": extraction_contract,
43
+ "effective_config": effective_config,
44
+ }
45
+ encoded = json.dumps(payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8")
46
+ return hashlib.sha256(encoded).hexdigest()
47
+
48
+
49
+ def require_matching_fingerprint(metadata: dict[str, object], fingerprint: str) -> str | None:
50
+ stored = metadata.get("execution_fingerprint")
51
+ if not isinstance(stored, str) or not stored:
52
+ return "execution_fingerprint_missing"
53
+ if stored != fingerprint:
54
+ return "execution_fingerprint_mismatch"
55
+ return None
56
+
57
+
58
+ @dataclass
59
+ class CompactOutputLock:
60
+ path: Path
61
+ run_id: str
62
+ _owner_token: str = field(default="", repr=False)
63
+ _guard_descriptor: int | None = field(default=None, repr=False)
64
+
65
+ @classmethod
66
+ def acquire(cls, out_dir: str | Path, run_id: str) -> "CompactOutputLock":
67
+ if not isinstance(run_id, str) or not run_id or len(run_id) > 128 or not run_id.isascii():
68
+ raise ValueError("compact lock requires a bounded ASCII run id")
69
+ path = Path(out_dir) / ".openmd-compact.lock"
70
+ path.parent.mkdir(parents=True, exist_ok=True)
71
+ # Never unlink the guard: replacing its inode would let two writers lock
72
+ # different files. The OS releases this descriptor's lock after a crash.
73
+ descriptor = os.open(path.parent / ".openmd-compact.guard", os.O_RDWR | os.O_CREAT, 0o600)
74
+ try:
75
+ _acquire_guard(descriptor, path)
76
+ existing = _read_owner(path)
77
+ token = uuid.uuid4().hex
78
+ owner = {"schema_version": COMPACT_LOCK_VERSION, "run_id": run_id, "owner_token": token}
79
+ _install_owner(path, owner, replace_existing=existing is not None)
80
+ return cls(path=path, run_id=run_id, _owner_token=token, _guard_descriptor=descriptor)
81
+ except BaseException:
82
+ os.close(descriptor)
83
+ raise
84
+
85
+ def release(self) -> None:
86
+ descriptor = self._guard_descriptor
87
+ if descriptor is None:
88
+ return
89
+ try:
90
+ owner = _read_owner(self.path)
91
+ if owner and owner["run_id"] == self.run_id and owner["owner_token"] == self._owner_token:
92
+ self.path.unlink()
93
+ except OSError:
94
+ # Preserve missing/changed/unknown ownership records. Closing the
95
+ # guard is still mandatory, including a failed normal cleanup.
96
+ pass
97
+ finally:
98
+ self._guard_descriptor = None
99
+ os.close(descriptor)
100
+
101
+
102
+ def _acquire_guard(descriptor: int, owner_path: Path) -> None:
103
+ try:
104
+ if os.name == "nt":
105
+ import msvcrt
106
+
107
+ os.lseek(descriptor, 0, os.SEEK_SET)
108
+ msvcrt.locking(descriptor, msvcrt.LK_NBLCK, 1)
109
+ else:
110
+ import fcntl
111
+
112
+ fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
113
+ except OSError as error:
114
+ if error.errno in {errno.EACCES, errno.EAGAIN, errno.EDEADLK}:
115
+ raise FileExistsError(errno.EEXIST, "compact output has a live owner", str(owner_path)) from None
116
+ raise
117
+
118
+
119
+ def _read_owner(path: Path) -> dict[str, str] | None:
120
+ try:
121
+ with path.open("rb") as stream:
122
+ raw = stream.read(4097)
123
+ except FileNotFoundError:
124
+ return None
125
+ try:
126
+ owner = json.loads(raw) if len(raw) <= 4096 else None
127
+ except (ValueError, UnicodeError):
128
+ owner = None
129
+ if (
130
+ not isinstance(owner, dict)
131
+ or set(owner) != {"schema_version", "run_id", "owner_token"}
132
+ or owner.get("schema_version") != COMPACT_LOCK_VERSION
133
+ or not isinstance(owner.get("run_id"), str)
134
+ or not 0 < len(owner["run_id"]) <= 128
135
+ or not owner["run_id"].isascii()
136
+ or not isinstance(owner.get("owner_token"), str)
137
+ or len(owner["owner_token"]) != 32
138
+ or any(char not in "0123456789abcdef" for char in owner["owner_token"])
139
+ ):
140
+ # Legacy UUID-only locks did not hold this guard, so acquiring it cannot
141
+ # establish that their owner is dead. Unknown/torn records also stay put.
142
+ raise FileExistsError(errno.EEXIST, "compact output has an unknown ownership record", str(path))
143
+ return owner
144
+
145
+
146
+ def _install_owner(path: Path, owner: dict[str, str], *, replace_existing: bool) -> None:
147
+ descriptor, temporary_name = tempfile.mkstemp(prefix=".openmd-compact-owner-", suffix=".tmp", dir=path.parent)
148
+ temporary = Path(temporary_name)
149
+ try:
150
+ with os.fdopen(descriptor, "wb") as stream:
151
+ stream.write(json.dumps(owner, separators=(",", ":")).encode("ascii"))
152
+ stream.flush()
153
+ os.fsync(stream.fileno())
154
+ if replace_existing:
155
+ # The guard proves this validated v2 owner is abandoned. Keep the
156
+ # canonical path present while replacing it so legacy O_EXCL users
157
+ # cannot slip into a remove/create gap.
158
+ os.replace(temporary, path)
159
+ else:
160
+ # Atomically publish a complete record without overwriting an old
161
+ # writer that may have created its legacy lock since our read.
162
+ os.link(temporary, path)
163
+ finally:
164
+ temporary.unlink(missing_ok=True)