modao-prd-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- modao_prd_cli/__init__.py +1 -0
- modao_prd_cli/modao_prd/__init__.py +4 -0
- modao_prd_cli/modao_prd/__main__.py +7 -0
- modao_prd_cli/modao_prd/browser.py +578 -0
- modao_prd_cli/modao_prd/capture_evidence.py +547 -0
- modao_prd_cli/modao_prd/classifier.py +96 -0
- modao_prd_cli/modao_prd/cli.py +205 -0
- modao_prd_cli/modao_prd/errors.py +31 -0
- modao_prd_cli/modao_prd/evidence.py +207 -0
- modao_prd_cli/modao_prd/explorer.py +300 -0
- modao_prd_cli/modao_prd/extractor.py +465 -0
- modao_prd_cli/modao_prd/models.py +56 -0
- modao_prd_cli/modao_prd/normalizer.py +135 -0
- modao_prd_cli/modao_prd/schemas/coverage-1.0.json +15 -0
- modao_prd_cli/modao_prd/schemas/document-2.0.json +21 -0
- modao_prd_cli/modao_prd/schemas/document-2.1.json +31 -0
- modao_prd_cli/modao_prd/schemas/manifest-1.0.json +28 -0
- modao_prd_cli/modao_prd/tests/__init__.py +1 -0
- modao_prd_cli/modao_prd/tests/fixtures/modao_sample.html +20 -0
- modao_prd_cli/modao_prd/tests/test_browser.py +54 -0
- modao_prd_cli/modao_prd/tests/test_classifier.py +32 -0
- modao_prd_cli/modao_prd/tests/test_cli.py +88 -0
- modao_prd_cli/modao_prd/tests/test_extractor.py +57 -0
- modao_prd_cli/modao_prd/tests/test_full_e2e.py +23 -0
- modao_prd_cli/modao_prd/tests/test_writers.py +79 -0
- modao_prd_cli/modao_prd/writers.py +468 -0
- modao_prd_cli-0.1.0.dist-info/METADATA +108 -0
- modao_prd_cli-0.1.0.dist-info/RECORD +32 -0
- modao_prd_cli-0.1.0.dist-info/WHEEL +5 -0
- modao_prd_cli-0.1.0.dist-info/entry_points.txt +2 -0
- modao_prd_cli-0.1.0.dist-info/licenses/LICENSE +22 -0
- modao_prd_cli-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,465 @@
|
|
|
1
|
+
"""Build the versioned Agent-friendly document model."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from datetime import datetime, timezone
|
|
7
|
+
from html.parser import HTMLParser
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
from urllib.parse import urlsplit
|
|
11
|
+
|
|
12
|
+
from .classifier import classify_blocks
|
|
13
|
+
from .capture_evidence import redact_url
|
|
14
|
+
from .normalizer import build_sections, clean_text, normalize_records
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
SCHEMA_VERSION = "2.1"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def build_document(raw: dict[str, Any]) -> dict[str, Any]:
|
|
21
|
+
blocks, _order_map = normalize_records(raw)
|
|
22
|
+
raw_screens = raw.get("screens") or [{"index": 0, "title": raw.get("title") or "页面 1"}]
|
|
23
|
+
raw_evidence = raw.get("evidence") or []
|
|
24
|
+
evidence_by_state: dict[str, list[str]] = {}
|
|
25
|
+
for item in raw_evidence:
|
|
26
|
+
if isinstance(item, dict) and item.get("state_id") and item.get("id"):
|
|
27
|
+
evidence_by_state.setdefault(str(item["state_id"]), []).append(str(item["id"]))
|
|
28
|
+
primary_evidence_by_state = _primary_evidence_by_state(raw_evidence)
|
|
29
|
+
raw_states = list(raw.get("states") or [{"id": "state-001", "url": raw.get("final_url"), "depth": 0}])
|
|
30
|
+
# The complete DOM/layout tree lives in evidence/state-*.dom.json. Keep
|
|
31
|
+
# document.json agent-readable by retaining state metadata and references,
|
|
32
|
+
# not duplicating the potentially very large tree inline.
|
|
33
|
+
states = [
|
|
34
|
+
{key: value for key, value in item.items() if key != "dom"}
|
|
35
|
+
for item in raw_states
|
|
36
|
+
]
|
|
37
|
+
state_by_id = {str(item.get("id")): item for item in states}
|
|
38
|
+
default_state_id = str(states[0].get("id") or "state-001")
|
|
39
|
+
screens = [
|
|
40
|
+
{
|
|
41
|
+
"id": f"screen-{index + 1:03d}",
|
|
42
|
+
"title": clean_text(str(item.get("title") or f"页面 {index + 1}")),
|
|
43
|
+
"source_id": item.get("source_id"),
|
|
44
|
+
"index": index,
|
|
45
|
+
"bounds": item.get("bounds") or {},
|
|
46
|
+
"state_id": str(item.get("state_id") or default_state_id),
|
|
47
|
+
"evidence_ids": list(primary_evidence_by_state.get(str(item.get("state_id") or default_state_id), [])),
|
|
48
|
+
}
|
|
49
|
+
for index, item in enumerate(raw_screens)
|
|
50
|
+
]
|
|
51
|
+
for block in blocks:
|
|
52
|
+
state_id = str(block.get("state_id") or default_state_id)
|
|
53
|
+
block["state_id"] = state_id
|
|
54
|
+
block["evidence_ids"] = list(block.get("evidence_ids") or primary_evidence_by_state.get(state_id, []))
|
|
55
|
+
sections = build_sections(screens, blocks)
|
|
56
|
+
tables = _build_tables(blocks)
|
|
57
|
+
block_by_id = {block.get("id"): block for block in blocks}
|
|
58
|
+
for table in tables:
|
|
59
|
+
source_block = block_by_id.get(table.get("source_block_id"), {})
|
|
60
|
+
table["state_id"] = source_block.get("state_id", default_state_id)
|
|
61
|
+
table["evidence_ids"] = list(source_block.get("evidence_ids") or primary_evidence_by_state.get(table["state_id"], []))
|
|
62
|
+
assets = [
|
|
63
|
+
{
|
|
64
|
+
"id": f"asset-{index + 1:03d}",
|
|
65
|
+
"block_id": block["id"],
|
|
66
|
+
**block.get("asset", {}),
|
|
67
|
+
}
|
|
68
|
+
for index, block in enumerate(blocks)
|
|
69
|
+
if block.get("type") == "image"
|
|
70
|
+
]
|
|
71
|
+
for asset in assets:
|
|
72
|
+
source_block = block_by_id.get(asset.get("block_id"), {})
|
|
73
|
+
asset["state_id"] = source_block.get("state_id", default_state_id)
|
|
74
|
+
asset["evidence_ids"] = list(source_block.get("evidence_ids") or primary_evidence_by_state.get(asset["state_id"], []))
|
|
75
|
+
requirements, interactions, warnings = classify_blocks(blocks)
|
|
76
|
+
for item in requirements + interactions:
|
|
77
|
+
item["method"] = "deterministic_text_classification"
|
|
78
|
+
item["confidence"] = "heuristic"
|
|
79
|
+
item["source_block_ids"] = list(item.get("source_block_ids", []))
|
|
80
|
+
source_blocks = [block_by_id[block_id] for block_id in item["source_block_ids"] if block_id in block_by_id]
|
|
81
|
+
state_ids = _unique([str(block.get("state_id") or default_state_id) for block in source_blocks])
|
|
82
|
+
screen_ids = _unique([str(block.get("screen_id") or "") for block in source_blocks if block.get("screen_id")])
|
|
83
|
+
evidence_ids = _unique(
|
|
84
|
+
evidence_id
|
|
85
|
+
for block in source_blocks
|
|
86
|
+
for evidence_id in block.get("evidence_ids", [])
|
|
87
|
+
)
|
|
88
|
+
if len(state_ids) == 1:
|
|
89
|
+
item["state_id"] = state_ids[0]
|
|
90
|
+
if len(screen_ids) == 1:
|
|
91
|
+
item["screen_id"] = screen_ids[0]
|
|
92
|
+
if evidence_ids:
|
|
93
|
+
item["evidence_ids"] = evidence_ids
|
|
94
|
+
observed_interactions = list(raw.get("observed_interactions") or raw.get("coverage", {}).get("transitions", []) or [])
|
|
95
|
+
for observed in observed_interactions:
|
|
96
|
+
target_id = str(observed.get("target_state_id") or observed.get("state_id") or default_state_id)
|
|
97
|
+
observed.setdefault("state_id", target_id)
|
|
98
|
+
observed.setdefault("screen_id", "screen-001")
|
|
99
|
+
observed.setdefault("evidence_ids", list(primary_evidence_by_state.get(target_id, [])))
|
|
100
|
+
warnings.extend(raw.get("warnings") or [])
|
|
101
|
+
warnings.extend(_document_warnings(raw, blocks))
|
|
102
|
+
warnings = _unique(warnings)
|
|
103
|
+
|
|
104
|
+
title = clean_text(str(raw.get("title") or raw.get("project_id") or "墨刀原型"))
|
|
105
|
+
source_url = redact_url(str(raw.get("requested_url") or raw.get("url") or ""))
|
|
106
|
+
quality = _quality(raw, blocks, screens)
|
|
107
|
+
stats = {
|
|
108
|
+
"screen_count": len(screens),
|
|
109
|
+
"section_count": len(sections),
|
|
110
|
+
"block_count": len(blocks),
|
|
111
|
+
"text_block_count": sum(block["type"] in {"text", "heading", "label", "tab"} for block in blocks),
|
|
112
|
+
"table_count": len(tables),
|
|
113
|
+
"image_count": len(assets),
|
|
114
|
+
"control_count": sum(block["type"] == "button" for block in blocks),
|
|
115
|
+
"requirement_count": len(requirements),
|
|
116
|
+
"interaction_count": len(interactions),
|
|
117
|
+
"warning_count": len(warnings),
|
|
118
|
+
"body_text_length": raw.get("body_text_length", 0),
|
|
119
|
+
"quality": quality,
|
|
120
|
+
"state_count": len(states),
|
|
121
|
+
"evidence_count": len(raw_evidence),
|
|
122
|
+
"network_count": len(raw.get("network") or []),
|
|
123
|
+
"visual_only_count": sum(int((item.get("evidence_stats") or {}).get("visual_only_count", 0)) for item in states),
|
|
124
|
+
}
|
|
125
|
+
facts = {
|
|
126
|
+
"screens": screens,
|
|
127
|
+
"states": states,
|
|
128
|
+
"blocks": blocks,
|
|
129
|
+
"tables": tables,
|
|
130
|
+
"assets": assets,
|
|
131
|
+
"observed_interactions": observed_interactions,
|
|
132
|
+
"visual_regions": _visual_regions(raw_states, primary_evidence_by_state),
|
|
133
|
+
}
|
|
134
|
+
derived = {
|
|
135
|
+
"requirements": requirements,
|
|
136
|
+
"declared_interactions": interactions,
|
|
137
|
+
}
|
|
138
|
+
coverage = {
|
|
139
|
+
"schema_version": "1.0",
|
|
140
|
+
**(raw.get("coverage") or {}),
|
|
141
|
+
"states": states,
|
|
142
|
+
"transitions": observed_interactions,
|
|
143
|
+
"uncovered": list((raw.get("coverage") or {}).get("uncovered", [])),
|
|
144
|
+
"quality": quality,
|
|
145
|
+
"summary": {
|
|
146
|
+
"state_count": len(states),
|
|
147
|
+
"transition_count": len(observed_interactions),
|
|
148
|
+
"evidence_count": len(raw_evidence),
|
|
149
|
+
"quality": quality,
|
|
150
|
+
},
|
|
151
|
+
"metrics": {
|
|
152
|
+
"dom": {
|
|
153
|
+
"states_captured": len(states),
|
|
154
|
+
"nodes_observed": sum(int((item.get("evidence_stats") or {}).get("dom_node_count", 0)) for item in states),
|
|
155
|
+
},
|
|
156
|
+
"page": {"screens_observed": len(screens)},
|
|
157
|
+
"scroll": {"attempted": bool(raw.get("capture"))},
|
|
158
|
+
"interaction": {
|
|
159
|
+
"actions_attempted": int((raw.get("coverage") or {}).get("actions_attempted", 0)),
|
|
160
|
+
"transitions_observed": len(observed_interactions),
|
|
161
|
+
},
|
|
162
|
+
"frame": {
|
|
163
|
+
"frames_observed": sum(int((item.get("evidence_stats") or {}).get("frame_count", 0)) for item in states),
|
|
164
|
+
},
|
|
165
|
+
"resource": {"network_records": len(raw.get("network") or [])},
|
|
166
|
+
"visual": {
|
|
167
|
+
"regions": sum(int((item.get("evidence_stats") or {}).get("visual_only_count", 0)) for item in states),
|
|
168
|
+
},
|
|
169
|
+
},
|
|
170
|
+
}
|
|
171
|
+
# Keep only a compact document overview here. The detailed blocks live in
|
|
172
|
+
# facts and the complete DOM/layout tree lives in evidence/state-*.dom.json.
|
|
173
|
+
# Do not duplicate blocks, rules, tables, assets or layout relations in the
|
|
174
|
+
# Agent-facing document: those projections used to make large prototypes
|
|
175
|
+
# needlessly expensive to read.
|
|
176
|
+
return {
|
|
177
|
+
"schema_version": SCHEMA_VERSION,
|
|
178
|
+
"source": {
|
|
179
|
+
"url": source_url,
|
|
180
|
+
"platform": "modao",
|
|
181
|
+
"access_mode": "public_share",
|
|
182
|
+
"title": title,
|
|
183
|
+
"fetched_at": datetime.now(timezone.utc).isoformat(),
|
|
184
|
+
"final_url": redact_url(str(raw.get("final_url") or source_url)),
|
|
185
|
+
"project_id": raw.get("project_id") or _project_id(source_url),
|
|
186
|
+
},
|
|
187
|
+
"capture": raw.get("capture") or {},
|
|
188
|
+
"facts": facts,
|
|
189
|
+
"derived": derived,
|
|
190
|
+
"evidence": raw_evidence,
|
|
191
|
+
"network": list(raw.get("network") or []),
|
|
192
|
+
"coverage": coverage,
|
|
193
|
+
"document": {
|
|
194
|
+
"title": title,
|
|
195
|
+
"type": "prototype",
|
|
196
|
+
"sections": sections,
|
|
197
|
+
},
|
|
198
|
+
"warnings": warnings,
|
|
199
|
+
"stats": stats,
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def summarize_document(document: dict[str, Any]) -> dict[str, Any]:
|
|
204
|
+
stats = document.get("stats", {})
|
|
205
|
+
return {
|
|
206
|
+
"schema_version": document.get("schema_version"),
|
|
207
|
+
"title": document.get("source", {}).get("title"),
|
|
208
|
+
"url": document.get("source", {}).get("url"),
|
|
209
|
+
"final_url": document.get("source", {}).get("final_url"),
|
|
210
|
+
"document_type": document.get("document", {}).get("type"),
|
|
211
|
+
"screens": stats.get("screen_count", len(document.get("facts", {}).get("screens", []))),
|
|
212
|
+
"sections": stats.get("section_count", 0),
|
|
213
|
+
"text_blocks": stats.get("text_block_count", 0),
|
|
214
|
+
"tables": stats.get("table_count", 0),
|
|
215
|
+
"images": stats.get("image_count", 0),
|
|
216
|
+
"controls": stats.get("control_count", 0),
|
|
217
|
+
"requirements": stats.get("requirement_count", 0),
|
|
218
|
+
"interactions": stats.get("interaction_count", 0),
|
|
219
|
+
"warnings": document.get("warnings", []),
|
|
220
|
+
"quality": stats.get("quality") or document.get("coverage", {}).get("quality"),
|
|
221
|
+
"states": stats.get("state_count", len(document.get("facts", {}).get("states", []))),
|
|
222
|
+
"evidence": stats.get("evidence_count", len(document.get("evidence", []))),
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def extract_from_html(html: str, *, title: str = "墨刀原型", url: str = "") -> dict[str, Any]:
|
|
227
|
+
"""Extract a small deterministic fixture without requiring a browser."""
|
|
228
|
+
|
|
229
|
+
parser = _FixtureHTMLParser()
|
|
230
|
+
parser.feed(html)
|
|
231
|
+
parser.close()
|
|
232
|
+
raw = {
|
|
233
|
+
"title": title,
|
|
234
|
+
"requested_url": url,
|
|
235
|
+
"final_url": url,
|
|
236
|
+
"project_id": _project_id(url),
|
|
237
|
+
"screens": parser.screens or [{"index": 0, "title": title}],
|
|
238
|
+
"records": parser.records,
|
|
239
|
+
"body_text_length": len(clean_text("".join(parser.body_text))),
|
|
240
|
+
}
|
|
241
|
+
return build_document(raw)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _build_tables(blocks: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
245
|
+
tables: list[dict[str, Any]] = []
|
|
246
|
+
for block in blocks:
|
|
247
|
+
if block.get("type") != "table":
|
|
248
|
+
continue
|
|
249
|
+
matrix = [
|
|
250
|
+
[clean_text(str(cell)) for cell in row]
|
|
251
|
+
for row in block.get("matrix", [])
|
|
252
|
+
if any(clean_text(str(cell)) for cell in row)
|
|
253
|
+
]
|
|
254
|
+
if not matrix:
|
|
255
|
+
continue
|
|
256
|
+
columns = _unique_columns(matrix[0])
|
|
257
|
+
rows = []
|
|
258
|
+
for values in matrix[1:]:
|
|
259
|
+
rows.append({column: values[index] if index < len(values) else "" for index, column in enumerate(columns)})
|
|
260
|
+
title = _table_title(blocks, block)
|
|
261
|
+
tables.append(
|
|
262
|
+
{
|
|
263
|
+
"id": f"table-{len(tables) + 1:03d}",
|
|
264
|
+
"title": title,
|
|
265
|
+
"screen_id": block.get("screen_id"),
|
|
266
|
+
"columns": columns,
|
|
267
|
+
"rows": rows,
|
|
268
|
+
"raw_matrix": matrix,
|
|
269
|
+
"cells": (block.get("table") or {}).get("rows", []),
|
|
270
|
+
"source_block_id": block["id"],
|
|
271
|
+
}
|
|
272
|
+
)
|
|
273
|
+
return tables
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _unique_columns(columns: list[str]) -> list[str]:
|
|
277
|
+
result: list[str] = []
|
|
278
|
+
counts: dict[str, int] = {}
|
|
279
|
+
for index, column in enumerate(columns):
|
|
280
|
+
name = column or f"column_{index + 1}"
|
|
281
|
+
counts[name] = counts.get(name, 0) + 1
|
|
282
|
+
result.append(name if counts[name] == 1 else f"{name}_{counts[name]}")
|
|
283
|
+
return result
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _table_title(blocks: list[dict[str, Any]], table_block: dict[str, Any]) -> str:
|
|
287
|
+
same_screen = [block for block in blocks if block.get("screen_id") == table_block.get("screen_id")]
|
|
288
|
+
position = next((index for index, block in enumerate(same_screen) if block["id"] == table_block["id"]), 0)
|
|
289
|
+
for block in reversed(same_screen[max(0, position - 3):position]):
|
|
290
|
+
text = clean_text(block.get("text"))
|
|
291
|
+
if block.get("type") in {"heading", "label", "tab"} and text:
|
|
292
|
+
return text.rstrip("::")
|
|
293
|
+
return "未命名表格"
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _document_warnings(raw: dict[str, Any], blocks: list[dict[str, Any]]) -> list[str]:
|
|
297
|
+
warnings: list[str] = []
|
|
298
|
+
if not raw.get("screens"):
|
|
299
|
+
warnings.append("未识别到明确画布节点,页面内容按默认页面归档。")
|
|
300
|
+
if not any(block.get("type") == "table" for block in blocks):
|
|
301
|
+
warnings.append("未识别到 HTML 表格;画布中的视觉表格可能仍保留在文本或图片中。")
|
|
302
|
+
return warnings
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def _quality(raw: dict[str, Any], blocks: list[dict[str, Any]], screens: list[dict[str, Any]]) -> str:
|
|
306
|
+
if not blocks or not screens:
|
|
307
|
+
return "unknown"
|
|
308
|
+
coverage = raw.get("coverage") or {}
|
|
309
|
+
if coverage.get("partial_reason") or any(
|
|
310
|
+
any(marker in str(item) for marker in ("未保存", "达到", "不可见", "无法读取", "未结构化"))
|
|
311
|
+
for item in raw.get("warnings", [])
|
|
312
|
+
):
|
|
313
|
+
return "partial"
|
|
314
|
+
return "complete_within_policy"
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def _primary_evidence_by_state(raw_evidence: list[dict[str, Any]]) -> dict[str, list[str]]:
|
|
318
|
+
"""Return the small set of evidence refs useful for Agent fact nodes.
|
|
319
|
+
|
|
320
|
+
Asset refs are intentionally kept at state level. Attaching every loaded
|
|
321
|
+
image to every block made a large page repeat the same list hundreds of
|
|
322
|
+
times. A block can still reach all state evidence through ``state_id`` and
|
|
323
|
+
the top-level evidence index.
|
|
324
|
+
"""
|
|
325
|
+
|
|
326
|
+
preferred_kinds = {"dom", "screen_screenshot"}
|
|
327
|
+
result: dict[str, list[str]] = {}
|
|
328
|
+
for item in raw_evidence:
|
|
329
|
+
if not isinstance(item, dict) or item.get("state_id") is None or not item.get("id"):
|
|
330
|
+
continue
|
|
331
|
+
if item.get("kind") not in preferred_kinds:
|
|
332
|
+
continue
|
|
333
|
+
state_id = str(item["state_id"])
|
|
334
|
+
refs = result.setdefault(state_id, [])
|
|
335
|
+
evidence_id = str(item["id"])
|
|
336
|
+
if evidence_id not in refs:
|
|
337
|
+
refs.append(evidence_id)
|
|
338
|
+
return result
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def _visual_regions(states: list[dict[str, Any]], evidence_by_state: dict[str, list[str]]) -> list[dict[str, Any]]:
|
|
342
|
+
regions: list[dict[str, Any]] = []
|
|
343
|
+
for state in states:
|
|
344
|
+
state_id = str(state.get("id") or "")
|
|
345
|
+
node_by_id = {node.get("id"): node for node in (state.get("dom") or {}).get("nodes", [])}
|
|
346
|
+
for region in (state.get("dom") or {}).get("visual_only", []):
|
|
347
|
+
node = node_by_id.get(region.get("node_id"), {})
|
|
348
|
+
regions.append(
|
|
349
|
+
{
|
|
350
|
+
**region,
|
|
351
|
+
"state_id": state_id,
|
|
352
|
+
"screen_id": node.get("screen_id") or "screen-001",
|
|
353
|
+
"evidence_ids": list(evidence_by_state.get(state_id, [])),
|
|
354
|
+
"classification": "visual_only",
|
|
355
|
+
}
|
|
356
|
+
)
|
|
357
|
+
return regions
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def _project_id(url: str) -> str:
|
|
361
|
+
match = re.search(r"/proto/([A-Za-z0-9_-]+)/sharing", url)
|
|
362
|
+
return match.group(1) if match else "unknown"
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def _unique(values: list[str]) -> list[str]:
|
|
366
|
+
result: list[str] = []
|
|
367
|
+
seen: set[str] = set()
|
|
368
|
+
for value in values:
|
|
369
|
+
if value not in seen:
|
|
370
|
+
seen.add(value)
|
|
371
|
+
result.append(value)
|
|
372
|
+
return result
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
class _FixtureHTMLParser(HTMLParser):
|
|
376
|
+
"""Small parser for offline fixtures; production extraction happens in JS."""
|
|
377
|
+
|
|
378
|
+
def __init__(self) -> None:
|
|
379
|
+
super().__init__(convert_charrefs=True)
|
|
380
|
+
self.stack: list[dict[str, Any]] = []
|
|
381
|
+
self.records: list[dict[str, Any]] = []
|
|
382
|
+
self.screens: list[dict[str, Any]] = []
|
|
383
|
+
self.body_text: list[str] = []
|
|
384
|
+
self._order = 0
|
|
385
|
+
|
|
386
|
+
def handle_starttag(self, tag: str, attrs_list: list[tuple[str, str | None]]) -> None:
|
|
387
|
+
attrs = dict(attrs_list)
|
|
388
|
+
classes = set((attrs.get("class") or "").split())
|
|
389
|
+
context = {"tag": tag, "classes": classes, "attrs": attrs, "text": [], "matrix": None}
|
|
390
|
+
self.stack.append(context)
|
|
391
|
+
if "mb-screen" in classes:
|
|
392
|
+
self.screens.append({"index": len(self.screens), "title": attrs.get("data-title") or f"页面 {len(self.screens) + 1}"})
|
|
393
|
+
if tag == "table":
|
|
394
|
+
context["matrix"] = []
|
|
395
|
+
if tag in {"td", "th"}:
|
|
396
|
+
context["cell"] = []
|
|
397
|
+
|
|
398
|
+
def handle_startendtag(self, tag: str, attrs_list: list[tuple[str, str | None]]) -> None:
|
|
399
|
+
if tag == "img":
|
|
400
|
+
attrs = dict(attrs_list)
|
|
401
|
+
self._add_record(
|
|
402
|
+
"image",
|
|
403
|
+
clean_text(attrs.get("alt")) or "image",
|
|
404
|
+
{"tag": tag, "classes": set(), "attrs": attrs},
|
|
405
|
+
src=attrs.get("src"),
|
|
406
|
+
alt=clean_text(attrs.get("alt")),
|
|
407
|
+
embedded=(attrs.get("src") or "").startswith("data:"),
|
|
408
|
+
natural_width=0,
|
|
409
|
+
natural_height=0,
|
|
410
|
+
)
|
|
411
|
+
return
|
|
412
|
+
self.handle_starttag(tag, attrs_list)
|
|
413
|
+
self.handle_endtag(tag)
|
|
414
|
+
|
|
415
|
+
def handle_endtag(self, tag: str) -> None:
|
|
416
|
+
if not self.stack:
|
|
417
|
+
return
|
|
418
|
+
index = len(self.stack) - 1
|
|
419
|
+
while index >= 0 and self.stack[index]["tag"] != tag:
|
|
420
|
+
index -= 1
|
|
421
|
+
if index < 0:
|
|
422
|
+
return
|
|
423
|
+
context = self.stack.pop(index)
|
|
424
|
+
text = clean_text("".join(context.get("text", [])))
|
|
425
|
+
if text and self.stack:
|
|
426
|
+
self.stack[-1].setdefault("text", []).append(text)
|
|
427
|
+
ancestor_classes = set().union(*(item.get("classes", set()) for item in self.stack)) if self.stack else set()
|
|
428
|
+
if tag == "tr" and self.stack:
|
|
429
|
+
table = next((item for item in reversed(self.stack) if item.get("tag") == "table"), None)
|
|
430
|
+
if table is not None:
|
|
431
|
+
table.setdefault("matrix", []).append(context.get("cells", []))
|
|
432
|
+
if tag in {"td", "th"} and self.stack:
|
|
433
|
+
row = next((item for item in reversed(self.stack) if item.get("tag") == "tr"), None)
|
|
434
|
+
if row is not None:
|
|
435
|
+
row.setdefault("cells", []).append(text)
|
|
436
|
+
if "wRichText" in context.get("classes", set()) and "wRichText" not in ancestor_classes and text:
|
|
437
|
+
self._add_record("text", text, context)
|
|
438
|
+
if tag == "table" and context.get("matrix"):
|
|
439
|
+
self._add_record("table", "\n".join(" | ".join(row) for row in context["matrix"]), context, matrix=context["matrix"])
|
|
440
|
+
if tag == "button" and text:
|
|
441
|
+
self._add_record("button", text, context)
|
|
442
|
+
|
|
443
|
+
def handle_data(self, data: str) -> None:
|
|
444
|
+
if not self.stack:
|
|
445
|
+
return
|
|
446
|
+
self.stack[-1].setdefault("text", []).append(data)
|
|
447
|
+
self.body_text.append(data)
|
|
448
|
+
|
|
449
|
+
def _add_record(self, kind: str, text: str, context: dict[str, Any], **extra: Any) -> None:
|
|
450
|
+
screen_index = max(
|
|
451
|
+
(index for index, item in enumerate(self.stack) if "mb-screen" in item.get("classes", set())),
|
|
452
|
+
default=0,
|
|
453
|
+
)
|
|
454
|
+
record = {
|
|
455
|
+
"kind": kind,
|
|
456
|
+
"text": text,
|
|
457
|
+
"screen_index": screen_index,
|
|
458
|
+
"tag": context.get("tag", ""),
|
|
459
|
+
"selector": "." + ".".join(sorted(context.get("classes", set()))) if context.get("classes") else context.get("tag", ""),
|
|
460
|
+
"bounds": {},
|
|
461
|
+
"dom_order": self._order,
|
|
462
|
+
**extra,
|
|
463
|
+
}
|
|
464
|
+
self._order += 1
|
|
465
|
+
self.records.append(record)
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Small, JSON-friendly data models for capture evidence and coverage."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass(slots=True)
|
|
10
|
+
class CaptureOptions:
|
|
11
|
+
"""Options shared by a capture run."""
|
|
12
|
+
|
|
13
|
+
max_item: int = 10 * 1024 * 1024
|
|
14
|
+
max_total: int = 200 * 1024 * 1024
|
|
15
|
+
force: bool = False
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(slots=True)
|
|
19
|
+
class EvidenceItem:
|
|
20
|
+
path: str
|
|
21
|
+
sha256: str
|
|
22
|
+
size: int
|
|
23
|
+
media_type: str
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(slots=True)
|
|
27
|
+
class CaptureResult:
|
|
28
|
+
output_dir: str
|
|
29
|
+
manifest: str
|
|
30
|
+
items: list[EvidenceItem] = field(default_factory=list)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass(slots=True)
|
|
34
|
+
class StateRecord:
|
|
35
|
+
id: str
|
|
36
|
+
title: str | None = None
|
|
37
|
+
url: str | None = None
|
|
38
|
+
screenshot: str | None = None
|
|
39
|
+
metadata: dict[str, Any] = field(default_factory=dict)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(slots=True)
|
|
43
|
+
class TransitionRecord:
|
|
44
|
+
source: str
|
|
45
|
+
target: str
|
|
46
|
+
trigger: str
|
|
47
|
+
action: str | None = None
|
|
48
|
+
evidence: list[str] = field(default_factory=list)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass(slots=True)
|
|
52
|
+
class CoverageReport:
|
|
53
|
+
states: list[StateRecord] = field(default_factory=list)
|
|
54
|
+
transitions: list[TransitionRecord] = field(default_factory=list)
|
|
55
|
+
uncovered: list[str] = field(default_factory=list)
|
|
56
|
+
summary: dict[str, Any] = field(default_factory=dict)
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
"""Turn browser records into stable, deterministic document blocks."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
SPACE_RE = re.compile(r"[ \t]+")
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def clean_text(value: str | None) -> str:
|
|
13
|
+
if not value:
|
|
14
|
+
return ""
|
|
15
|
+
value = value.replace("\u00a0", " ").replace("\r\n", "\n").replace("\r", "\n")
|
|
16
|
+
value = "\n".join(SPACE_RE.sub(" ", line).strip() for line in value.split("\n"))
|
|
17
|
+
return value.strip()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def infer_block_type(kind: str, text: str, screen_title: str = "") -> str:
|
|
21
|
+
if kind == "table":
|
|
22
|
+
return "table"
|
|
23
|
+
if kind == "button":
|
|
24
|
+
return "button"
|
|
25
|
+
if kind == "image":
|
|
26
|
+
return "image"
|
|
27
|
+
if re.match(r"^(TAB\s*\d+|页面\s*\d+)[-—::]", text, re.I):
|
|
28
|
+
return "tab"
|
|
29
|
+
if screen_title and text == screen_title:
|
|
30
|
+
return "heading"
|
|
31
|
+
if len(text) <= 32 and (text.endswith((":", ":")) or text in {"攻略", "奖励", "记录", "其他说明"}):
|
|
32
|
+
return "label"
|
|
33
|
+
return "text"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def normalize_records(raw: dict[str, Any]) -> tuple[list[dict[str, Any]], dict[int, str]]:
|
|
37
|
+
"""Normalize raw browser records and return blocks plus raw-order mapping."""
|
|
38
|
+
|
|
39
|
+
screens = raw.get("screens") or [{"index": 0, "title": "页面 1"}]
|
|
40
|
+
screen_by_index = {int(item.get("index", index)): item for index, item in enumerate(screens)}
|
|
41
|
+
blocks: list[dict[str, Any]] = []
|
|
42
|
+
raw_order_to_block: dict[int, str] = {}
|
|
43
|
+
seen: dict[tuple[Any, ...], str] = {}
|
|
44
|
+
|
|
45
|
+
records = sorted(raw.get("records") or [], key=lambda item: int(item.get("dom_order", 0)))
|
|
46
|
+
for raw_record in records:
|
|
47
|
+
text = clean_text(raw_record.get("text"))
|
|
48
|
+
kind = str(raw_record.get("kind", "text"))
|
|
49
|
+
if not text and kind != "image":
|
|
50
|
+
continue
|
|
51
|
+
screen_index = int(raw_record.get("screen_index", 0))
|
|
52
|
+
screen = screen_by_index.get(screen_index, {"title": f"页面 {screen_index + 1}"})
|
|
53
|
+
screen_id = f"screen-{screen_index + 1:03d}"
|
|
54
|
+
block_type = infer_block_type(kind, text, str(screen.get("title", "")))
|
|
55
|
+
bounds = raw_record.get("bounds") or {}
|
|
56
|
+
dedupe_key = (
|
|
57
|
+
screen_id,
|
|
58
|
+
block_type,
|
|
59
|
+
text,
|
|
60
|
+
bounds.get("x"),
|
|
61
|
+
bounds.get("y"),
|
|
62
|
+
raw_record.get("selector"),
|
|
63
|
+
)
|
|
64
|
+
block_id = f"block-{len(blocks) + 1:03d}"
|
|
65
|
+
block: dict[str, Any] = {
|
|
66
|
+
"id": block_id,
|
|
67
|
+
"type": block_type,
|
|
68
|
+
"screen_id": screen_id,
|
|
69
|
+
"order": len(blocks) + 1,
|
|
70
|
+
"text": text,
|
|
71
|
+
"source": {
|
|
72
|
+
"selector": raw_record.get("selector", ""),
|
|
73
|
+
"tag": raw_record.get("tag", ""),
|
|
74
|
+
"dom_order": raw_record.get("dom_order"),
|
|
75
|
+
},
|
|
76
|
+
"state_id": str(raw_record.get("state_id") or "state-001"),
|
|
77
|
+
}
|
|
78
|
+
if dedupe_key in seen:
|
|
79
|
+
block["duplicate_of"] = seen[dedupe_key]
|
|
80
|
+
else:
|
|
81
|
+
seen[dedupe_key] = block_id
|
|
82
|
+
if bounds:
|
|
83
|
+
block["bounds"] = bounds
|
|
84
|
+
if kind == "table":
|
|
85
|
+
block["matrix"] = raw_record.get("matrix") or []
|
|
86
|
+
block["table"] = raw_record.get("table") or {}
|
|
87
|
+
if kind == "image":
|
|
88
|
+
block["asset"] = {
|
|
89
|
+
"src": raw_record.get("src"),
|
|
90
|
+
"alt": raw_record.get("alt", ""),
|
|
91
|
+
"embedded": bool(raw_record.get("embedded", False)),
|
|
92
|
+
"natural_width": raw_record.get("natural_width", 0),
|
|
93
|
+
"natural_height": raw_record.get("natural_height", 0),
|
|
94
|
+
}
|
|
95
|
+
blocks.append(block)
|
|
96
|
+
raw_order = raw_record.get("dom_order")
|
|
97
|
+
if isinstance(raw_order, int):
|
|
98
|
+
raw_order_to_block[raw_order] = block_id
|
|
99
|
+
|
|
100
|
+
return blocks, raw_order_to_block
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def build_sections(
|
|
104
|
+
screens: list[dict[str, Any]], blocks: list[dict[str, Any]]
|
|
105
|
+
) -> list[dict[str, Any]]:
|
|
106
|
+
"""Create lightweight page sections without inventing semantic content."""
|
|
107
|
+
|
|
108
|
+
sections: list[dict[str, Any]] = []
|
|
109
|
+
for index, screen in enumerate(screens):
|
|
110
|
+
screen_id = f"screen-{index + 1:03d}"
|
|
111
|
+
screen_blocks = [block for block in blocks if block["screen_id"] == screen_id]
|
|
112
|
+
tab_blocks = [block for block in screen_blocks if block["type"] == "tab"]
|
|
113
|
+
if not tab_blocks:
|
|
114
|
+
sections.append(
|
|
115
|
+
{
|
|
116
|
+
"id": f"section-{len(sections) + 1:03d}",
|
|
117
|
+
"title": screen.get("title") or f"页面 {index + 1}",
|
|
118
|
+
"screen_id": screen_id,
|
|
119
|
+
"block_ids": [block["id"] for block in screen_blocks],
|
|
120
|
+
}
|
|
121
|
+
)
|
|
122
|
+
continue
|
|
123
|
+
|
|
124
|
+
for tab_index, tab in enumerate(tab_blocks):
|
|
125
|
+
start = screen_blocks.index(tab)
|
|
126
|
+
end = screen_blocks.index(tab_blocks[tab_index + 1]) if tab_index + 1 < len(tab_blocks) else len(screen_blocks)
|
|
127
|
+
sections.append(
|
|
128
|
+
{
|
|
129
|
+
"id": f"section-{len(sections) + 1:03d}",
|
|
130
|
+
"title": tab["text"],
|
|
131
|
+
"screen_id": screen_id,
|
|
132
|
+
"block_ids": [block["id"] for block in screen_blocks[start:end]],
|
|
133
|
+
}
|
|
134
|
+
)
|
|
135
|
+
return sections
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "coverage-1.0.json",
|
|
4
|
+
"title": "Capture coverage report",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"required": ["schema_version", "states", "transitions"],
|
|
7
|
+
"properties": {
|
|
8
|
+
"schema_version": {"const": "1.0"},
|
|
9
|
+
"states": {"type": "array"},
|
|
10
|
+
"transitions": {"type": "array"},
|
|
11
|
+
"uncovered": {"type": "array", "items": {"type": "string"}},
|
|
12
|
+
"summary": {"type": "object"}
|
|
13
|
+
},
|
|
14
|
+
"additionalProperties": true
|
|
15
|
+
}
|