openmd-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- openmd/__init__.py +3 -0
- openmd/__main__.py +5 -0
- openmd/adapters/__init__.py +1 -0
- openmd/adapters/base.py +95 -0
- openmd/adapters/bilibili/__init__.py +11 -0
- openmd/adapters/bilibili/adapter.py +104 -0
- openmd/adapters/bilibili/discovery.py +33 -0
- openmd/adapters/douyin/__init__.py +11 -0
- openmd/adapters/douyin/adapter.py +117 -0
- openmd/adapters/douyin/discovery.py +36 -0
- openmd/adapters/douyin/sidecar.py +77 -0
- openmd/adapters/generic_media/__init__.py +14 -0
- openmd/adapters/high_privacy.py +77 -0
- openmd/adapters/local_audio/__init__.py +14 -0
- openmd/adapters/local_audio/adapter.py +169 -0
- openmd/adapters/local_audio/asr.py +58 -0
- openmd/adapters/local_audio/backends/__init__.py +1 -0
- openmd/adapters/local_audio/backends/auto.py +177 -0
- openmd/adapters/local_audio/backends/faster_whisper.py +285 -0
- openmd/adapters/local_audio/backends/funasr_sense_voice.py +14 -0
- openmd/adapters/local_audio/backends/qwen3_asr.py +254 -0
- openmd/adapters/opencli.py +770 -0
- openmd/adapters/pdf/__init__.py +11 -0
- openmd/adapters/pdf/adapter.py +153 -0
- openmd/adapters/pdf/backends/__init__.py +1 -0
- openmd/adapters/pdf/backends/markitdown.py +165 -0
- openmd/adapters/pdf/parser.py +56 -0
- openmd/adapters/qq_group/__init__.py +11 -0
- openmd/adapters/qq_group/adapter.py +289 -0
- openmd/adapters/qq_group/parsers/__init__.py +1 -0
- openmd/adapters/qq_group/parsers/html.py +60 -0
- openmd/adapters/qq_group/parsers/json.py +71 -0
- openmd/adapters/registry.py +149 -0
- openmd/adapters/screenshot/__init__.py +11 -0
- openmd/adapters/screenshot/adapter.py +154 -0
- openmd/adapters/screenshot/backends/__init__.py +1 -0
- openmd/adapters/screenshot/backends/paddle.py +15 -0
- openmd/adapters/screenshot/backends/rapidocr.py +176 -0
- openmd/adapters/screenshot/backends/tesseract.py +15 -0
- openmd/adapters/screenshot/ocr.py +84 -0
- openmd/adapters/wechat_group/__init__.py +11 -0
- openmd/adapters/wechat_group/adapter.py +160 -0
- openmd/adapters/wechat_group/wx_cli.py +268 -0
- openmd/adapters/wechat_moments/__init__.py +11 -0
- openmd/adapters/wechat_moments/adapter.py +167 -0
- openmd/adapters/wechat_moments/wx_cli.py +265 -0
- openmd/adapters/weixin/__init__.py +11 -0
- openmd/adapters/weixin/adapter.py +148 -0
- openmd/adapters/weixin/discovery.py +510 -0
- openmd/adapters/weixin/report.py +278 -0
- openmd/adapters/xiaohongshu.py +158 -0
- openmd/backends/__init__.py +1 -0
- openmd/backends/base.py +204 -0
- openmd/backends/bilibili.py +1480 -0
- openmd/backends/bilibili_wbi.py +277 -0
- openmd/backends/discourse.py +426 -0
- openmd/backends/douyin.py +868 -0
- openmd/backends/douyin_detail.py +124 -0
- openmd/backends/douyin_providers.py +602 -0
- openmd/backends/executor.py +546 -0
- openmd/backends/feishu.py +861 -0
- openmd/backends/generic_media.py +77 -0
- openmd/backends/interactive.py +60 -0
- openmd/backends/ocr.py +41 -0
- openmd/backends/opencli.py +82 -0
- openmd/backends/pdf.py +122 -0
- openmd/backends/registry.py +234 -0
- openmd/backends/screenshot.py +79 -0
- openmd/backends/video_metadata.py +167 -0
- openmd/backends/web_article.py +363 -0
- openmd/backends/wechat_group.py +35 -0
- openmd/backends/wechat_moments.py +35 -0
- openmd/backends/weixin.py +767 -0
- openmd/backends/x.py +455 -0
- openmd/backends/xiaoe.py +940 -0
- openmd/backends/xiaohongshu.py +911 -0
- openmd/backends/xiaohongshu_providers.py +876 -0
- openmd/backends/xiaoyuzhou.py +1017 -0
- openmd/backends/youtube.py +280 -0
- openmd/backends/youtube_discovery.py +1245 -0
- openmd/backends/youtube_transcript.py +213 -0
- openmd/browser_extension/background.js +231 -0
- openmd/browser_extension/icons/icon-128.png +0 -0
- openmd/browser_extension/icons/icon-16.png +0 -0
- openmd/browser_extension/icons/icon-32.png +0 -0
- openmd/browser_extension/icons/icon-48.png +0 -0
- openmd/browser_extension/icons/icon-512.png +0 -0
- openmd/browser_extension/idle.html +6 -0
- openmd/browser_extension/manifest.json +43 -0
- openmd/browser_extension/options.html +42 -0
- openmd/browser_extension/options.js +117 -0
- openmd/browser_extension/popup.css +71 -0
- openmd/browser_extension/popup.html +17 -0
- openmd/browser_extension/popup.js +59 -0
- openmd/browser_extension/vendor/opencli-bridge/LICENSE +190 -0
- openmd/browser_extension/vendor/opencli-bridge/VENDOR.md +31 -0
- openmd/browser_extension/vendor/opencli-bridge/background.js +2742 -0
- openmd/browser_host.py +503 -0
- openmd/browser_setup.py +106 -0
- openmd/cli.py +2012 -0
- openmd/console.py +32 -0
- openmd/core/__init__.py +1 -0
- openmd/core/capabilities.py +162 -0
- openmd/core/content_validation.py +323 -0
- openmd/core/coverage.py +163 -0
- openmd/core/detector.py +247 -0
- openmd/core/doctor.py +104 -0
- openmd/core/execution_policy.py +164 -0
- openmd/core/filename.py +64 -0
- openmd/core/input_validation.py +91 -0
- openmd/core/local_config.py +96 -0
- openmd/core/manifest.py +156 -0
- openmd/core/models.py +168 -0
- openmd/core/opencli_output.py +34 -0
- openmd/core/operation_contract.py +60 -0
- openmd/core/output.py +194 -0
- openmd/core/package_contract.py +335 -0
- openmd/core/package_validator.py +489 -0
- openmd/core/package_writer.py +666 -0
- openmd/core/planner.py +345 -0
- openmd/core/preferences.py +61 -0
- openmd/core/readable.py +129 -0
- openmd/core/receipt.py +285 -0
- openmd/core/renderer.py +185 -0
- openmd/core/runlog.py +208 -0
- openmd/core/secrets.py +313 -0
- openmd/doctor_view.py +68 -0
- openmd/interactive.py +453 -0
- openmd/ocr/__init__.py +1 -0
- openmd/ocr/breakers.py +84 -0
- openmd/ocr/config.py +54 -0
- openmd/ocr/documents.py +522 -0
- openmd/ocr/endpoint.py +72 -0
- openmd/ocr/layout.py +92 -0
- openmd/ocr/layout_filter.py +118 -0
- openmd/ocr/layout_worker.py +89 -0
- openmd/ocr/native_math.py +220 -0
- openmd/ocr/office.py +305 -0
- openmd/ocr/ollama.py +200 -0
- openmd/ocr/otsl.py +51 -0
- openmd/ocr/ovis-prompt.txt +1 -0
- openmd/ocr/pipeline.py +384 -0
- openmd/ocr/settings.json +25 -0
- openmd/ocr/tiling.py +631 -0
- openmd/post_video.py +154 -0
- openmd/probe.py +397 -0
- openmd/routing/__init__.py +1 -0
- openmd/routing/cards/builtin.json +620 -0
- openmd/routing/cards/defuddle.json +70 -0
- openmd/routing/cards/x-tweet-fetcher.json +71 -0
- openmd/routing/cards/xyz-dl.json +60 -0
- openmd/routing/cards/yutto.json +68 -0
- openmd/routing/entry_shim.py +71 -0
- openmd/routing/lifecycle.py +459 -0
- openmd/routing/loader.py +138 -0
- openmd/routing/providers.py +158 -0
- openmd/routing/readers.py +108 -0
- openmd/routing/routes.json +504 -0
- openmd/routing/runner.py +192 -0
- openmd/routing/view.py +40 -0
- openmd/runtime/__init__.py +1 -0
- openmd/runtime/_windows_job.py +234 -0
- openmd/runtime/asr.py +217 -0
- openmd/runtime/asr_worker.py +119 -0
- openmd/runtime/gpu_lock.py +81 -0
- openmd/runtime/gpu_quarantine.py +85 -0
- openmd/runtime/interpreters.py +407 -0
- openmd/runtime/managed_tools.py +204 -0
- openmd/runtime/models.py +202 -0
- openmd/runtime/services.py +175 -0
- openmd/runtime/tempdirs.py +227 -0
- openmd_cli-0.1.0.dist-info/METADATA +168 -0
- openmd_cli-0.1.0.dist-info/RECORD +177 -0
- openmd_cli-0.1.0.dist-info/WHEEL +5 -0
- openmd_cli-0.1.0.dist-info/entry_points.txt +2 -0
- openmd_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
- openmd_cli-0.1.0.dist-info/top_level.txt +1 -0
openmd/core/coverage.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"""Small reusable counters for collection coverage summaries.
|
|
2
|
+
|
|
3
|
+
The helper intentionally keeps only bounded enums and integer counts. It does
|
|
4
|
+
not retain URLs, output paths, backend payloads, or other acquisition data, so
|
|
5
|
+
platform summaries can safely serialize the result without leaking transient
|
|
6
|
+
locators.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from collections import Counter, defaultdict
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from typing import Any, Iterable, Mapping
|
|
14
|
+
|
|
15
|
+
from .models import ContentItem, ExtractionComponent, TranscriptResult
|
|
16
|
+
from .package_contract import package_source_level
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
_ITEM_STATUSES = ("verified", "partial", "failed")
|
|
20
|
+
_BLOCKING_COMPONENT_STATUSES = {"partial", "failed", "retryable", "auth_challenge"}
|
|
21
|
+
_CONTENT_TYPES = ("text", "image_text", "video", "mixed")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class CollectionCoverage:
|
|
26
|
+
"""Accumulate item outcomes and component provenance without identifiers."""
|
|
27
|
+
|
|
28
|
+
attempted: int = 0
|
|
29
|
+
skipped_existing: int = 0
|
|
30
|
+
terminal_gap_items: int = 0
|
|
31
|
+
retryable_items: int = 0
|
|
32
|
+
_item_statuses: Counter[str] = field(default_factory=Counter)
|
|
33
|
+
_component_statuses: dict[str, Counter[str]] = field(
|
|
34
|
+
default_factory=lambda: defaultdict(Counter)
|
|
35
|
+
)
|
|
36
|
+
_component_sources: dict[str, Counter[str]] = field(
|
|
37
|
+
default_factory=lambda: defaultdict(Counter)
|
|
38
|
+
)
|
|
39
|
+
_source_levels: Counter[str] = field(default_factory=Counter)
|
|
40
|
+
|
|
41
|
+
def record_result(
|
|
42
|
+
self,
|
|
43
|
+
result: TranscriptResult,
|
|
44
|
+
*,
|
|
45
|
+
skipped_existing: bool = False,
|
|
46
|
+
force_failed: bool = False,
|
|
47
|
+
) -> None:
|
|
48
|
+
self.record(
|
|
49
|
+
status="failed" if force_failed else result.status,
|
|
50
|
+
components=result.normalized_components,
|
|
51
|
+
skipped_existing=skipped_existing,
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
def record_metadata(
|
|
55
|
+
self,
|
|
56
|
+
metadata: Mapping[str, Any],
|
|
57
|
+
*,
|
|
58
|
+
skipped_existing: bool = False,
|
|
59
|
+
) -> None:
|
|
60
|
+
raw_components = metadata.get("components")
|
|
61
|
+
components = raw_components if isinstance(raw_components, list) else []
|
|
62
|
+
self.record(
|
|
63
|
+
status=str(metadata.get("status") or "failed"),
|
|
64
|
+
components=components,
|
|
65
|
+
skipped_existing=skipped_existing,
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
def record_failure(self) -> None:
|
|
69
|
+
self.record(status="failed", components=[])
|
|
70
|
+
|
|
71
|
+
def record(
|
|
72
|
+
self,
|
|
73
|
+
*,
|
|
74
|
+
status: str,
|
|
75
|
+
components: Iterable[ExtractionComponent | Mapping[str, Any]],
|
|
76
|
+
skipped_existing: bool = False,
|
|
77
|
+
) -> None:
|
|
78
|
+
normalized_status = status if status in _ITEM_STATUSES else "failed"
|
|
79
|
+
self.attempted += 1
|
|
80
|
+
self._item_statuses[normalized_status] += 1
|
|
81
|
+
if skipped_existing:
|
|
82
|
+
self.skipped_existing += 1
|
|
83
|
+
|
|
84
|
+
has_terminal_gap = False
|
|
85
|
+
has_blocking_component = normalized_status == "failed"
|
|
86
|
+
for component in components:
|
|
87
|
+
component_id, component_status, source_level = _component_projection(component)
|
|
88
|
+
if not component_id:
|
|
89
|
+
continue
|
|
90
|
+
source_level = package_source_level(source_level)
|
|
91
|
+
self._component_statuses[component_id][component_status] += 1
|
|
92
|
+
self._component_sources[component_id][source_level] += 1
|
|
93
|
+
self._source_levels[source_level] += 1
|
|
94
|
+
has_terminal_gap = has_terminal_gap or component_status == "unavailable"
|
|
95
|
+
has_blocking_component = (
|
|
96
|
+
has_blocking_component
|
|
97
|
+
or component_status in _BLOCKING_COMPONENT_STATUSES
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
if has_terminal_gap:
|
|
101
|
+
self.terminal_gap_items += 1
|
|
102
|
+
if has_blocking_component:
|
|
103
|
+
self.retryable_items += 1
|
|
104
|
+
|
|
105
|
+
def as_dict(self) -> dict[str, Any]:
|
|
106
|
+
component_counts: dict[str, dict[str, int]] = {}
|
|
107
|
+
component_source_counts: dict[str, dict[str, int]] = {}
|
|
108
|
+
for component_id in sorted(self._component_statuses):
|
|
109
|
+
statuses = self._component_statuses[component_id]
|
|
110
|
+
component_counts[component_id] = {
|
|
111
|
+
"total": sum(statuses.values()),
|
|
112
|
+
**{key: int(value) for key, value in sorted(statuses.items())},
|
|
113
|
+
}
|
|
114
|
+
component_source_counts[component_id] = {
|
|
115
|
+
key: int(value)
|
|
116
|
+
for key, value in sorted(self._component_sources[component_id].items())
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
return {
|
|
120
|
+
"attempted": self.attempted,
|
|
121
|
+
"verified": int(self._item_statuses["verified"]),
|
|
122
|
+
"partial": int(self._item_statuses["partial"]),
|
|
123
|
+
"failed": int(self._item_statuses["failed"]),
|
|
124
|
+
"skipped_existing": self.skipped_existing,
|
|
125
|
+
"terminal_gaps": sum(
|
|
126
|
+
counts["unavailable"] for counts in self._component_statuses.values()
|
|
127
|
+
),
|
|
128
|
+
"terminal_gap_items": self.terminal_gap_items,
|
|
129
|
+
"retryable": sum(
|
|
130
|
+
counts["retryable"] for counts in self._component_statuses.values()
|
|
131
|
+
),
|
|
132
|
+
"retryable_items": self.retryable_items,
|
|
133
|
+
"component_counts": component_counts,
|
|
134
|
+
"component_source_counts": component_source_counts,
|
|
135
|
+
"source_counts": {
|
|
136
|
+
key: int(value) for key, value in sorted(self._source_levels.items())
|
|
137
|
+
},
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def collection_content_type_counts(items: Iterable[ContentItem]) -> dict[str, int]:
|
|
142
|
+
"""Return stable text/image_text/video/mixed counts for discovered items."""
|
|
143
|
+
counts = Counter({key: 0 for key in _CONTENT_TYPES})
|
|
144
|
+
for item in items:
|
|
145
|
+
raw_kind = item.raw_meta.get("media_kind") if isinstance(item.raw_meta, dict) else None
|
|
146
|
+
kind = str(raw_kind or item.source_type or "mixed").strip().lower().replace("-", "_")
|
|
147
|
+
if kind in {"article", "document", "text_first", "text_only"}:
|
|
148
|
+
kind = "text"
|
|
149
|
+
if kind not in _CONTENT_TYPES:
|
|
150
|
+
kind = "mixed"
|
|
151
|
+
counts[kind] += 1
|
|
152
|
+
return {key: int(counts[key]) for key in _CONTENT_TYPES}
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _component_projection(
|
|
156
|
+
component: ExtractionComponent | Mapping[str, Any],
|
|
157
|
+
) -> tuple[str, str, str]:
|
|
158
|
+
if isinstance(component, ExtractionComponent):
|
|
159
|
+
return component.id, component.status, component.source_level
|
|
160
|
+
component_id = str(component.get("id") or "").strip()
|
|
161
|
+
status = str(component.get("status") or "failed").strip()
|
|
162
|
+
source_level = str(component.get("source_level") or "partial_extraction").strip()
|
|
163
|
+
return component_id, status, source_level
|
openmd/core/detector.py
ADDED
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
"""Conservative URL platform detector for Openmd V0 categories."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from urllib.parse import unquote, urlparse
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
PLATFORM_IDS: tuple[str, ...] = (
|
|
11
|
+
"bilibili",
|
|
12
|
+
"youtube",
|
|
13
|
+
"weixin",
|
|
14
|
+
"wechat_group",
|
|
15
|
+
"wechat_moments",
|
|
16
|
+
"qq_group",
|
|
17
|
+
"xiaoyuzhou",
|
|
18
|
+
"douyin",
|
|
19
|
+
"tiktok",
|
|
20
|
+
"xiaohongshu",
|
|
21
|
+
"xiaoe",
|
|
22
|
+
"x",
|
|
23
|
+
"feishu",
|
|
24
|
+
"web_article",
|
|
25
|
+
"generic_media",
|
|
26
|
+
"pdf",
|
|
27
|
+
"screenshot",
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
MEDIA_EXTENSIONS: tuple[str, ...] = (
|
|
31
|
+
".mp4",
|
|
32
|
+
".m4v",
|
|
33
|
+
".mov",
|
|
34
|
+
".webm",
|
|
35
|
+
".mkv",
|
|
36
|
+
".mp3",
|
|
37
|
+
".m4a",
|
|
38
|
+
".wav",
|
|
39
|
+
".flac",
|
|
40
|
+
".m3u8",
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
# Keep the legacy `pdf` platform identity for the approved PDF/Office lane.
|
|
44
|
+
DOCUMENT_EXTENSIONS: tuple[str, ...] = (".pdf", ".docx", ".pptx", ".xlsx")
|
|
45
|
+
|
|
46
|
+
IMAGE_TEXT_EXTENSIONS: tuple[str, ...] = (
|
|
47
|
+
".png",
|
|
48
|
+
".jpg",
|
|
49
|
+
".jpeg",
|
|
50
|
+
".webp",
|
|
51
|
+
".bmp",
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
QQ_EXPORT_EXTENSIONS: tuple[str, ...] = (".json", ".html", ".txt")
|
|
55
|
+
|
|
56
|
+
_BV_RE = re.compile(r"^BV[0-9A-Za-z]+$")
|
|
57
|
+
_URL_RE = re.compile(r"https?://[^\s,。]+")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def extract_first_url(input_value: str) -> str:
|
|
61
|
+
value = input_value.strip()
|
|
62
|
+
match = _URL_RE.search(value)
|
|
63
|
+
if not match:
|
|
64
|
+
return value
|
|
65
|
+
return match.group(0).rstrip(".,,。;;!!))]")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _host_and_path(input_value: str) -> tuple[str, str]:
|
|
69
|
+
value = extract_first_url(input_value)
|
|
70
|
+
parse_value = value if "://" in value else f"https://{value}"
|
|
71
|
+
parsed = urlparse(parse_value)
|
|
72
|
+
return parsed.netloc.lower(), parsed.path.lower()
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _host_endswith(host: str, suffixes: tuple[str, ...]) -> bool:
|
|
76
|
+
return any(host == suffix or host.endswith(f".{suffix}") for suffix in suffixes)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def looks_like_media_url(input_value: str) -> bool:
|
|
80
|
+
_, path = _host_and_path(input_value)
|
|
81
|
+
return path.endswith(MEDIA_EXTENSIONS)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _path_for_extension_check(input_value: str) -> str:
|
|
85
|
+
value = extract_first_url(input_value)
|
|
86
|
+
parsed = urlparse(value)
|
|
87
|
+
if parsed.scheme == "file":
|
|
88
|
+
return parsed.path.lower()
|
|
89
|
+
if parsed.scheme in {"http", "https"}:
|
|
90
|
+
return parsed.path.lower()
|
|
91
|
+
return value.lower()
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def detect_platform(input_value: str, *, explicit_media: bool = False) -> str:
|
|
95
|
+
value = input_value.strip()
|
|
96
|
+
if explicit_media:
|
|
97
|
+
return "generic_media"
|
|
98
|
+
directory_platform = _local_directory_platform(value)
|
|
99
|
+
if directory_platform is not None:
|
|
100
|
+
return directory_platform
|
|
101
|
+
if _is_weixin_official_account_selector(value):
|
|
102
|
+
return "weixin"
|
|
103
|
+
if _is_wechat_moments_selector(value):
|
|
104
|
+
return "wechat_moments"
|
|
105
|
+
if _is_wechat_group_selector(value):
|
|
106
|
+
return "wechat_group"
|
|
107
|
+
if _is_qq_group_selector(value) or _looks_like_qq_export_path(value):
|
|
108
|
+
return "qq_group"
|
|
109
|
+
if _BV_RE.match(value):
|
|
110
|
+
return "bilibili"
|
|
111
|
+
if re.fullmatch(r"\d{15,20}", value):
|
|
112
|
+
return "douyin"
|
|
113
|
+
|
|
114
|
+
extension_path = _path_for_extension_check(value)
|
|
115
|
+
if extension_path.endswith(DOCUMENT_EXTENSIONS):
|
|
116
|
+
return "pdf"
|
|
117
|
+
if extension_path.endswith(IMAGE_TEXT_EXTENSIONS):
|
|
118
|
+
return "screenshot"
|
|
119
|
+
if extension_path.endswith(MEDIA_EXTENSIONS):
|
|
120
|
+
return "generic_media"
|
|
121
|
+
|
|
122
|
+
host, path = _host_and_path(value)
|
|
123
|
+
if _host_endswith(host, ("bilibili.com", "b23.tv")):
|
|
124
|
+
return "bilibili"
|
|
125
|
+
if _host_endswith(host, ("youtube.com", "youtu.be", "youtube-nocookie.com")):
|
|
126
|
+
return "youtube"
|
|
127
|
+
if host == "mp.weixin.qq.com":
|
|
128
|
+
return "weixin"
|
|
129
|
+
if _host_endswith(host, ("xiaoyuzhoufm.com",)):
|
|
130
|
+
return "xiaoyuzhou"
|
|
131
|
+
if _host_endswith(host, ("douyin.com", "iesdouyin.com")):
|
|
132
|
+
return "douyin"
|
|
133
|
+
if _host_endswith(host, ("tiktok.com",)):
|
|
134
|
+
return "tiktok"
|
|
135
|
+
if _host_endswith(host, ("xiaohongshu.com", "xhslink.com")):
|
|
136
|
+
return "xiaohongshu"
|
|
137
|
+
if _host_endswith(host, ("xet.citv.cn", "xet.pomoho.com", "xiaoeknow.com")) or "xiaoe" in host:
|
|
138
|
+
return "xiaoe"
|
|
139
|
+
if _host_endswith(host, ("x.com", "twitter.com")):
|
|
140
|
+
return "x"
|
|
141
|
+
if _host_endswith(host, ("feishu.cn", "larksuite.com")):
|
|
142
|
+
return "feishu"
|
|
143
|
+
if path.endswith(MEDIA_EXTENSIONS):
|
|
144
|
+
return "generic_media"
|
|
145
|
+
return "web_article"
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _local_directory_platform(value: str) -> str | None:
|
|
149
|
+
path = _local_path_for_detection(value)
|
|
150
|
+
if path is None or not path.is_dir():
|
|
151
|
+
return None
|
|
152
|
+
|
|
153
|
+
seen_pdf = False
|
|
154
|
+
seen_image = False
|
|
155
|
+
seen_media = False
|
|
156
|
+
checked = 0
|
|
157
|
+
try:
|
|
158
|
+
for candidate in path.rglob("*"):
|
|
159
|
+
if not candidate.is_file():
|
|
160
|
+
continue
|
|
161
|
+
suffix = candidate.suffix.lower()
|
|
162
|
+
seen_pdf = seen_pdf or suffix in DOCUMENT_EXTENSIONS
|
|
163
|
+
seen_image = seen_image or suffix in IMAGE_TEXT_EXTENSIONS
|
|
164
|
+
seen_media = seen_media or suffix in MEDIA_EXTENSIONS
|
|
165
|
+
checked += 1
|
|
166
|
+
if checked >= 200 or seen_pdf:
|
|
167
|
+
break
|
|
168
|
+
except OSError:
|
|
169
|
+
return None
|
|
170
|
+
|
|
171
|
+
if seen_pdf:
|
|
172
|
+
return "pdf"
|
|
173
|
+
if seen_image:
|
|
174
|
+
return "screenshot"
|
|
175
|
+
if seen_media:
|
|
176
|
+
return "generic_media"
|
|
177
|
+
return None
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _local_path_for_detection(value: str) -> Path | None:
|
|
181
|
+
parsed = urlparse(value)
|
|
182
|
+
if parsed.scheme in {"http", "https"}:
|
|
183
|
+
return None
|
|
184
|
+
if len(parsed.scheme) == 1 and len(value) >= 3 and value[1] == ":" and value[2] in {"\\", "/"}:
|
|
185
|
+
return Path(value)
|
|
186
|
+
if parsed.scheme == "file":
|
|
187
|
+
raw_path = unquote(parsed.path)
|
|
188
|
+
if parsed.netloc and not raw_path:
|
|
189
|
+
raw_path = parsed.netloc
|
|
190
|
+
if raw_path.startswith("/") and len(raw_path) >= 4 and raw_path[2] == ":":
|
|
191
|
+
raw_path = raw_path[1:]
|
|
192
|
+
return Path(raw_path)
|
|
193
|
+
if parsed.scheme:
|
|
194
|
+
return None
|
|
195
|
+
return Path(value)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _is_wechat_group_selector(value: str) -> bool:
|
|
199
|
+
parsed = urlparse(value.strip())
|
|
200
|
+
if parsed.scheme in {"wx", "wechat"} and parsed.netloc.lower() == "group":
|
|
201
|
+
return bool(parsed.path.strip("/"))
|
|
202
|
+
if parsed.scheme in {"wx-group", "wechat-group"}:
|
|
203
|
+
return bool(parsed.netloc or parsed.path.strip("/"))
|
|
204
|
+
return False
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _is_weixin_official_account_selector(value: str) -> bool:
|
|
208
|
+
parsed = urlparse(value.strip())
|
|
209
|
+
if parsed.scheme in {"weixin", "wx-official", "wechat-official"}:
|
|
210
|
+
return bool(parsed.netloc or parsed.path.strip("/"))
|
|
211
|
+
if parsed.scheme == "wechat" and parsed.netloc.lower() in {
|
|
212
|
+
"official",
|
|
213
|
+
"official-account",
|
|
214
|
+
"oa",
|
|
215
|
+
"author",
|
|
216
|
+
"account",
|
|
217
|
+
"fakeid",
|
|
218
|
+
"biz",
|
|
219
|
+
}:
|
|
220
|
+
return bool(parsed.path.strip("/") or parsed.query.strip())
|
|
221
|
+
return False
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _is_wechat_moments_selector(value: str) -> bool:
|
|
225
|
+
parsed = urlparse(value.strip())
|
|
226
|
+
if parsed.scheme in {"wx", "wechat"} and parsed.netloc.lower() in {"moments", "sns"}:
|
|
227
|
+
return bool(parsed.path.strip("/"))
|
|
228
|
+
if parsed.scheme in {"wx-moments", "wechat-moments"}:
|
|
229
|
+
return bool(parsed.netloc or parsed.path.strip("/"))
|
|
230
|
+
return False
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _is_qq_group_selector(value: str) -> bool:
|
|
234
|
+
parsed = urlparse(value.strip())
|
|
235
|
+
if parsed.scheme == "qq" and parsed.netloc.lower() == "group":
|
|
236
|
+
return bool(parsed.path.strip("/"))
|
|
237
|
+
if parsed.scheme in {"qq-group", "qq_export", "qq-export"}:
|
|
238
|
+
return bool(parsed.netloc or parsed.path.strip("/"))
|
|
239
|
+
return False
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _looks_like_qq_export_path(value: str) -> bool:
|
|
243
|
+
extension_path = _path_for_extension_check(value)
|
|
244
|
+
if not extension_path.endswith(QQ_EXPORT_EXTENSIONS):
|
|
245
|
+
return False
|
|
246
|
+
filename = extension_path.replace("\\", "/").rsplit("/", 1)[-1]
|
|
247
|
+
return "qq" in filename or "group-chat" in filename
|
openmd/core/doctor.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""Safe local-only doctor projections for Openmd capability profiles."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import importlib.util
|
|
6
|
+
import shutil
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from .capabilities import validate_capability_manifest
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
DOCTOR_PROFILES = ("all", "core", "web", "media", "documents", "authenticated-social", "tools")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _module_available(name: str) -> bool:
|
|
16
|
+
try:
|
|
17
|
+
return importlib.util.find_spec(name) is not None
|
|
18
|
+
except (ImportError, ValueError):
|
|
19
|
+
return False
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _media_diagnostics() -> dict[str, Any]:
|
|
23
|
+
from openmd.runtime.interpreters import select_asr_runtime
|
|
24
|
+
from openmd.runtime.models import configured_models
|
|
25
|
+
|
|
26
|
+
runtime = select_asr_runtime()
|
|
27
|
+
ffmpeg_available = shutil.which("ffmpeg") is not None
|
|
28
|
+
ready = bool(runtime.probe.get("ready")) and ffmpeg_available
|
|
29
|
+
next_action = "Local CUDA ASR runtime and models are ready" if ready else runtime.reason
|
|
30
|
+
if not ffmpeg_available:
|
|
31
|
+
next_action = "Install ffmpeg and make it available on PATH, then rerun openmd doctor --profile media"
|
|
32
|
+
return {
|
|
33
|
+
"ffmpeg_on_path": ffmpeg_available,
|
|
34
|
+
"local_asr_dependency": _module_available("faster_whisper"),
|
|
35
|
+
"asr_runtime": runtime.as_dict(),
|
|
36
|
+
"models": {key: model.as_dict() for key, model in configured_models().items()},
|
|
37
|
+
"ready": ready,
|
|
38
|
+
"next_action": next_action,
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _document_diagnostics() -> dict[str, Any]:
|
|
43
|
+
from openmd.ocr.config import settings
|
|
44
|
+
from openmd.ocr.layout import probe_layout
|
|
45
|
+
from openmd.ocr.office import soffice_path
|
|
46
|
+
from openmd.ocr.ollama import probe_ollama
|
|
47
|
+
from openmd.runtime.gpu_lock import gpu_lock_path
|
|
48
|
+
from openmd.runtime.gpu_quarantine import quarantine_path
|
|
49
|
+
|
|
50
|
+
try:
|
|
51
|
+
ollama = probe_ollama(settings())
|
|
52
|
+
except ValueError as exc:
|
|
53
|
+
ollama = {"reachable": False, "ready": False, "models": {}, "error_code": getattr(exc, "code", "ocr_setting_invalid")}
|
|
54
|
+
layout = probe_layout()
|
|
55
|
+
layout.update({"device": "cpu", "cpu_only": True, "cuda_visible_policy": "reject", "gpu_lease_inherited": False})
|
|
56
|
+
dependencies = {"pymupdf_available": _module_available("pymupdf"), "pillow_available": _module_available("PIL"), "numpy_available": _module_available("numpy")}
|
|
57
|
+
ready = all(dependencies.values()) and ollama["ready"] and layout["ready"]
|
|
58
|
+
return {"markitdown_available": _module_available("markitdown"), "ocr_dependency_available": _module_available("rapidocr_onnxruntime"), **dependencies, "ollama": ollama, "layout_runtime": layout, "libreoffice_available": soffice_path() is not None, "gpu_lock": {"path": str(gpu_lock_path()), "implementation": "OS file lock shared with ASR", "crash_recovery": "OS releases lease; persistent file is reused", "quarantine_exists": quarantine_path().exists(), "quarantine_recovery": "read-only wait for empty /api/ps; never unload other clients", "marker_only_reset": "openmd doctor --clear-gpu-quarantine"}, "ready": ready, "network_access": "configured_ollama_metadata_only", "next_action": "Document OCR dependencies are ready" if ready else "Start Ollama yourself and provide existing models / configure ocr_python and model_roots; Openmd does not install, download, pull or start services."}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def build_doctor_report(profile: str) -> dict[str, Any]:
|
|
62
|
+
if profile not in DOCTOR_PROFILES:
|
|
63
|
+
raise ValueError(f"unsupported doctor profile: {profile}")
|
|
64
|
+
try:
|
|
65
|
+
validate_capability_manifest()
|
|
66
|
+
manifest_valid = True
|
|
67
|
+
except ValueError:
|
|
68
|
+
manifest_valid = False
|
|
69
|
+
diagnostics: dict[str, Any] = {
|
|
70
|
+
"profile": profile,
|
|
71
|
+
"manifest_valid": manifest_valid,
|
|
72
|
+
"network_access": "not_probed",
|
|
73
|
+
"private_session_state": "not_read",
|
|
74
|
+
"account_actions": "not_performed",
|
|
75
|
+
"ready": manifest_valid,
|
|
76
|
+
}
|
|
77
|
+
if profile == "all":
|
|
78
|
+
diagnostics.update(_media_diagnostics())
|
|
79
|
+
diagnostics["openmd_on_path"] = shutil.which("openmd") is not None
|
|
80
|
+
diagnostics["ready"] = bool(diagnostics["ready"] and manifest_valid)
|
|
81
|
+
diagnostics["optional_profiles"] = {
|
|
82
|
+
"web": {"opencli_on_path": shutil.which("opencli") is not None, "network_gate": "explicit_opt_in"},
|
|
83
|
+
"documents": _document_diagnostics(),
|
|
84
|
+
}
|
|
85
|
+
elif profile == "core":
|
|
86
|
+
diagnostics.update({"openmd_on_path": shutil.which("openmd") is not None})
|
|
87
|
+
elif profile == "web":
|
|
88
|
+
diagnostics.update({"opencli_on_path": shutil.which("opencli") is not None, "network_gate": "explicit_opt_in"})
|
|
89
|
+
elif profile == "media":
|
|
90
|
+
diagnostics.update(_media_diagnostics())
|
|
91
|
+
diagnostics["ready"] = bool(diagnostics["ready"] and manifest_valid)
|
|
92
|
+
elif profile == "documents":
|
|
93
|
+
diagnostics.update(_document_diagnostics())
|
|
94
|
+
diagnostics["ready"] = manifest_valid and diagnostics["ready"]
|
|
95
|
+
elif profile == "tools":
|
|
96
|
+
# Populated by the tool-card diagnostics without executing a command.
|
|
97
|
+
from openmd.routing.providers import doctor_tools
|
|
98
|
+
|
|
99
|
+
tool_report = doctor_tools()
|
|
100
|
+
diagnostics["tools"] = tool_report
|
|
101
|
+
diagnostics["ready"] = manifest_valid and bool(tool_report.get("ok", False))
|
|
102
|
+
else:
|
|
103
|
+
diagnostics.update({"session_owner": "browser_platform_or_sidecar", "private_session_probe": False})
|
|
104
|
+
return diagnostics
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"""Execution identity and compact package writer ownership rules."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import errno
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import tempfile
|
|
10
|
+
import uuid
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
FINGERPRINT_CONTRACT_VERSION = "openmd.low-token.v1"
|
|
16
|
+
COMPACT_LOCK_VERSION = "openmd.compact-lock.v2"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def execution_fingerprint(
|
|
20
|
+
*,
|
|
21
|
+
requested_profile: str | None,
|
|
22
|
+
requested_modalities: tuple[str, ...],
|
|
23
|
+
provider_chain: tuple[str, ...],
|
|
24
|
+
backend_override: str | None,
|
|
25
|
+
model: str | None,
|
|
26
|
+
language: str | None,
|
|
27
|
+
authorization_flags: dict[str, bool] | None = None,
|
|
28
|
+
temporary_url: str | None = None,
|
|
29
|
+
extraction_contract: dict[str, object] | None = None,
|
|
30
|
+
effective_config: dict[str, object] | None = None,
|
|
31
|
+
) -> str:
|
|
32
|
+
"""Return a stable fingerprint without credentials, gates, or transient locators."""
|
|
33
|
+
del authorization_flags, temporary_url
|
|
34
|
+
payload = {
|
|
35
|
+
"contract_version": FINGERPRINT_CONTRACT_VERSION,
|
|
36
|
+
"requested_profile": requested_profile,
|
|
37
|
+
"requested_modalities": list(requested_modalities),
|
|
38
|
+
"provider_chain": list(provider_chain),
|
|
39
|
+
"backend_override": backend_override,
|
|
40
|
+
"model": model,
|
|
41
|
+
"language": language,
|
|
42
|
+
"extraction_contract": extraction_contract,
|
|
43
|
+
"effective_config": effective_config,
|
|
44
|
+
}
|
|
45
|
+
encoded = json.dumps(payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8")
|
|
46
|
+
return hashlib.sha256(encoded).hexdigest()
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def require_matching_fingerprint(metadata: dict[str, object], fingerprint: str) -> str | None:
|
|
50
|
+
stored = metadata.get("execution_fingerprint")
|
|
51
|
+
if not isinstance(stored, str) or not stored:
|
|
52
|
+
return "execution_fingerprint_missing"
|
|
53
|
+
if stored != fingerprint:
|
|
54
|
+
return "execution_fingerprint_mismatch"
|
|
55
|
+
return None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@dataclass
|
|
59
|
+
class CompactOutputLock:
|
|
60
|
+
path: Path
|
|
61
|
+
run_id: str
|
|
62
|
+
_owner_token: str = field(default="", repr=False)
|
|
63
|
+
_guard_descriptor: int | None = field(default=None, repr=False)
|
|
64
|
+
|
|
65
|
+
@classmethod
|
|
66
|
+
def acquire(cls, out_dir: str | Path, run_id: str) -> "CompactOutputLock":
|
|
67
|
+
if not isinstance(run_id, str) or not run_id or len(run_id) > 128 or not run_id.isascii():
|
|
68
|
+
raise ValueError("compact lock requires a bounded ASCII run id")
|
|
69
|
+
path = Path(out_dir) / ".openmd-compact.lock"
|
|
70
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
71
|
+
# Never unlink the guard: replacing its inode would let two writers lock
|
|
72
|
+
# different files. The OS releases this descriptor's lock after a crash.
|
|
73
|
+
descriptor = os.open(path.parent / ".openmd-compact.guard", os.O_RDWR | os.O_CREAT, 0o600)
|
|
74
|
+
try:
|
|
75
|
+
_acquire_guard(descriptor, path)
|
|
76
|
+
existing = _read_owner(path)
|
|
77
|
+
token = uuid.uuid4().hex
|
|
78
|
+
owner = {"schema_version": COMPACT_LOCK_VERSION, "run_id": run_id, "owner_token": token}
|
|
79
|
+
_install_owner(path, owner, replace_existing=existing is not None)
|
|
80
|
+
return cls(path=path, run_id=run_id, _owner_token=token, _guard_descriptor=descriptor)
|
|
81
|
+
except BaseException:
|
|
82
|
+
os.close(descriptor)
|
|
83
|
+
raise
|
|
84
|
+
|
|
85
|
+
def release(self) -> None:
|
|
86
|
+
descriptor = self._guard_descriptor
|
|
87
|
+
if descriptor is None:
|
|
88
|
+
return
|
|
89
|
+
try:
|
|
90
|
+
owner = _read_owner(self.path)
|
|
91
|
+
if owner and owner["run_id"] == self.run_id and owner["owner_token"] == self._owner_token:
|
|
92
|
+
self.path.unlink()
|
|
93
|
+
except OSError:
|
|
94
|
+
# Preserve missing/changed/unknown ownership records. Closing the
|
|
95
|
+
# guard is still mandatory, including a failed normal cleanup.
|
|
96
|
+
pass
|
|
97
|
+
finally:
|
|
98
|
+
self._guard_descriptor = None
|
|
99
|
+
os.close(descriptor)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _acquire_guard(descriptor: int, owner_path: Path) -> None:
|
|
103
|
+
try:
|
|
104
|
+
if os.name == "nt":
|
|
105
|
+
import msvcrt
|
|
106
|
+
|
|
107
|
+
os.lseek(descriptor, 0, os.SEEK_SET)
|
|
108
|
+
msvcrt.locking(descriptor, msvcrt.LK_NBLCK, 1)
|
|
109
|
+
else:
|
|
110
|
+
import fcntl
|
|
111
|
+
|
|
112
|
+
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
113
|
+
except OSError as error:
|
|
114
|
+
if error.errno in {errno.EACCES, errno.EAGAIN, errno.EDEADLK}:
|
|
115
|
+
raise FileExistsError(errno.EEXIST, "compact output has a live owner", str(owner_path)) from None
|
|
116
|
+
raise
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _read_owner(path: Path) -> dict[str, str] | None:
|
|
120
|
+
try:
|
|
121
|
+
with path.open("rb") as stream:
|
|
122
|
+
raw = stream.read(4097)
|
|
123
|
+
except FileNotFoundError:
|
|
124
|
+
return None
|
|
125
|
+
try:
|
|
126
|
+
owner = json.loads(raw) if len(raw) <= 4096 else None
|
|
127
|
+
except (ValueError, UnicodeError):
|
|
128
|
+
owner = None
|
|
129
|
+
if (
|
|
130
|
+
not isinstance(owner, dict)
|
|
131
|
+
or set(owner) != {"schema_version", "run_id", "owner_token"}
|
|
132
|
+
or owner.get("schema_version") != COMPACT_LOCK_VERSION
|
|
133
|
+
or not isinstance(owner.get("run_id"), str)
|
|
134
|
+
or not 0 < len(owner["run_id"]) <= 128
|
|
135
|
+
or not owner["run_id"].isascii()
|
|
136
|
+
or not isinstance(owner.get("owner_token"), str)
|
|
137
|
+
or len(owner["owner_token"]) != 32
|
|
138
|
+
or any(char not in "0123456789abcdef" for char in owner["owner_token"])
|
|
139
|
+
):
|
|
140
|
+
# Legacy UUID-only locks did not hold this guard, so acquiring it cannot
|
|
141
|
+
# establish that their owner is dead. Unknown/torn records also stay put.
|
|
142
|
+
raise FileExistsError(errno.EEXIST, "compact output has an unknown ownership record", str(path))
|
|
143
|
+
return owner
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _install_owner(path: Path, owner: dict[str, str], *, replace_existing: bool) -> None:
|
|
147
|
+
descriptor, temporary_name = tempfile.mkstemp(prefix=".openmd-compact-owner-", suffix=".tmp", dir=path.parent)
|
|
148
|
+
temporary = Path(temporary_name)
|
|
149
|
+
try:
|
|
150
|
+
with os.fdopen(descriptor, "wb") as stream:
|
|
151
|
+
stream.write(json.dumps(owner, separators=(",", ":")).encode("ascii"))
|
|
152
|
+
stream.flush()
|
|
153
|
+
os.fsync(stream.fileno())
|
|
154
|
+
if replace_existing:
|
|
155
|
+
# The guard proves this validated v2 owner is abandoned. Keep the
|
|
156
|
+
# canonical path present while replacing it so legacy O_EXCL users
|
|
157
|
+
# cannot slip into a remove/create gap.
|
|
158
|
+
os.replace(temporary, path)
|
|
159
|
+
else:
|
|
160
|
+
# Atomically publish a complete record without overwriting an old
|
|
161
|
+
# writer that may have created its legacy lock since our read.
|
|
162
|
+
os.link(temporary, path)
|
|
163
|
+
finally:
|
|
164
|
+
temporary.unlink(missing_ok=True)
|