mendwright 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mendwright/__init__.py +92 -0
- mendwright/__main__.py +6 -0
- mendwright/aio.py +686 -0
- mendwright/capture.py +162 -0
- mendwright/cli.py +689 -0
- mendwright/client.py +216 -0
- mendwright/coordinator.py +289 -0
- mendwright/demo/__init__.py +35 -0
- mendwright/demo/_page.py +171 -0
- mendwright/demo/_story.py +470 -0
- mendwright/demo/v1.html +29 -0
- mendwright/demo/v2.html +33 -0
- mendwright/drivers/__init__.py +25 -0
- mendwright/drivers/base.py +316 -0
- mendwright/drivers/playwright_async.py +148 -0
- mendwright/drivers/playwright_sync.py +193 -0
- mendwright/drivers/registry.py +37 -0
- mendwright/drivers/selenium.py +596 -0
- mendwright/exceptions.py +10 -0
- mendwright/facade.py +783 -0
- mendwright/gate.py +62 -0
- mendwright/journal.py +180 -0
- mendwright/limits.py +236 -0
- mendwright/prompt.py +67 -0
- mendwright/providers/__init__.py +24 -0
- mendwright/providers/_errors.py +376 -0
- mendwright/providers/anthropic.py +177 -0
- mendwright/providers/base.py +25 -0
- mendwright/providers/bedrock.py +126 -0
- mendwright/providers/detect.py +170 -0
- mendwright/providers/gateway.py +69 -0
- mendwright/providers/gemini.py +171 -0
- mendwright/providers/litellm.py +176 -0
- mendwright/providers/openai_compat.py +255 -0
- mendwright/publish.py +74 -0
- mendwright/py.typed +0 -0
- mendwright/pytest_plugin.py +505 -0
- mendwright/resolver.py +133 -0
- mendwright/roles.py +232 -0
- mendwright/selectors.py +181 -0
- mendwright/store.py +722 -0
- mendwright/types.py +56 -0
- mendwright/validation.py +98 -0
- mendwright-0.3.0.dist-info/METADATA +282 -0
- mendwright-0.3.0.dist-info/RECORD +49 -0
- mendwright-0.3.0.dist-info/WHEEL +5 -0
- mendwright-0.3.0.dist-info/entry_points.txt +5 -0
- mendwright-0.3.0.dist-info/licenses/LICENSE +21 -0
- mendwright-0.3.0.dist-info/top_level.txt +1 -0
mendwright/cli.py
ADDED
|
@@ -0,0 +1,689 @@
|
|
|
1
|
+
"""The `mendwright` command (also `python -m mendwright`).
|
|
2
|
+
|
|
3
|
+
mendwright demo watch a broken selector get healed, offline
|
|
4
|
+
mendwright doctor check the install, browser libraries, provider and store
|
|
5
|
+
mendwright report list the selectors and heals in an override store
|
|
6
|
+
mendwright --version
|
|
7
|
+
|
|
8
|
+
main(argv=None) returns the exit code instead of exiting, so it can be called from tests.
|
|
9
|
+
Nothing here imports a browser library or calls a model, except `mendwright demo --live`,
|
|
10
|
+
which asks the provider from the environment.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import argparse
|
|
16
|
+
import importlib.metadata
|
|
17
|
+
import importlib.util
|
|
18
|
+
import json
|
|
19
|
+
import os
|
|
20
|
+
import sys
|
|
21
|
+
import textwrap
|
|
22
|
+
from urllib.parse import urlsplit
|
|
23
|
+
|
|
24
|
+
from .demo._story import BROWSERS, write_line
|
|
25
|
+
|
|
26
|
+
DEFAULT_STORE_PATH = './.mendwright/overrides.json' # the same default mendwright.open() uses
|
|
27
|
+
|
|
28
|
+
BROWSER_LIBRARIES = ('playwright', 'selenium', 'patchright', 'camoufox')
|
|
29
|
+
|
|
30
|
+
# Variables auto_provider() and the providers read. doctor prints their names, never values.
|
|
31
|
+
PROVIDER_VARS = (
|
|
32
|
+
'MENDWRIGHT_PROVIDER', 'ANTHROPIC_API_KEY', 'ANTHROPIC_MODEL', 'ANTHROPIC_BASE_URL',
|
|
33
|
+
'OPENAI_API_KEY', 'OPENAI_MODEL', 'OPENAI_BASE_URL', 'GEMINI_API_KEY', 'GEMINI_MODEL',
|
|
34
|
+
'MENDWRIGHT_GATEWAY', 'MENDWRIGHT_KEY', 'MENDWRIGHT_AUTH_HEADER', 'OLLAMA_MODEL',
|
|
35
|
+
'OLLAMA_HOST',
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
# MENDWRIGHT_* variables whose value is safe to print. Anything else shows as hidden.
|
|
39
|
+
_SHOWN_VALUES = ('MENDWRIGHT_STORE', 'MENDWRIGHT_TIMEOUT_MS', 'MENDWRIGHT_DRY_RUN',
|
|
40
|
+
'MENDWRIGHT_DISABLE', 'MENDWRIGHT_PROVIDER', 'MENDWRIGHT_AUTH_HEADER')
|
|
41
|
+
_SECRET_WORDS = ('KEY', 'TOKEN', 'SECRET', 'PASSWORD', 'PASS', 'AUTH', 'CREDENTIAL')
|
|
42
|
+
|
|
43
|
+
REPORT_FORMATS = ('text', 'md', 'json')
|
|
44
|
+
LAST_HEALS = 10
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _version():
|
|
48
|
+
from . import __version__
|
|
49
|
+
return __version__
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _out(text=''):
|
|
53
|
+
write_line(text, sys.stdout)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _err(text):
|
|
57
|
+
write_line(text, sys.stderr)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _wrap(text, indent=' ', width=88):
|
|
61
|
+
return textwrap.fill(text, width=width, initial_indent=indent, subsequent_indent=indent,
|
|
62
|
+
break_long_words=False, break_on_hyphens=False)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
# --- doctor ----------------------------------------------------------------------------
|
|
66
|
+
|
|
67
|
+
def _scrub(text):
|
|
68
|
+
"""text with the value of every secret-looking environment variable masked."""
|
|
69
|
+
if not isinstance(text, str):
|
|
70
|
+
return text
|
|
71
|
+
for name, value in os.environ.items():
|
|
72
|
+
upper = name.upper()
|
|
73
|
+
if value and len(value) >= 4 and any(word in upper for word in _SECRET_WORDS):
|
|
74
|
+
text = text.replace(value, '***')
|
|
75
|
+
return text
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _origin(url):
|
|
79
|
+
"""scheme://host[:port] of a URL, without user info, path or query."""
|
|
80
|
+
try:
|
|
81
|
+
parts = urlsplit(url if '://' in url else 'https://' + url)
|
|
82
|
+
host = parts.hostname or ''
|
|
83
|
+
port = ':' + str(parts.port) if parts.port else ''
|
|
84
|
+
return (parts.scheme + '://' + host + port) if host else '(set, not a URL)'
|
|
85
|
+
except Exception:
|
|
86
|
+
return '(set, not a URL)'
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _endpoint(url):
|
|
90
|
+
"""scheme://host[:port]/path of a provider's base URL, without user info or query.
|
|
91
|
+
None when there is no URL. The host is not a secret, and shows where the page goes."""
|
|
92
|
+
if not isinstance(url, str) or not url.strip():
|
|
93
|
+
return None
|
|
94
|
+
try:
|
|
95
|
+
parts = urlsplit(url if '://' in url else 'https://' + url)
|
|
96
|
+
host = parts.hostname or ''
|
|
97
|
+
port = ':' + str(parts.port) if parts.port else ''
|
|
98
|
+
return (parts.scheme + '://' + host + port + parts.path.rstrip('/')) if host else None
|
|
99
|
+
except Exception:
|
|
100
|
+
return None
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _dist_version(name):
|
|
104
|
+
try:
|
|
105
|
+
return importlib.metadata.version(name)
|
|
106
|
+
except Exception:
|
|
107
|
+
return None
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _browser_libraries():
|
|
111
|
+
"""Which browser libraries are on the import path. Found with find_spec, so nothing
|
|
112
|
+
is imported or launched."""
|
|
113
|
+
rows = []
|
|
114
|
+
for name in BROWSER_LIBRARIES:
|
|
115
|
+
try:
|
|
116
|
+
found = importlib.util.find_spec(name) is not None
|
|
117
|
+
except Exception:
|
|
118
|
+
found = False
|
|
119
|
+
rows.append({'name': name, 'installed': found,
|
|
120
|
+
'version': _dist_version(name) if found else None})
|
|
121
|
+
return rows
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _flag_problem(name):
|
|
125
|
+
from .facade import _env_flag
|
|
126
|
+
try:
|
|
127
|
+
return None, _env_flag(name)
|
|
128
|
+
except ValueError as exc:
|
|
129
|
+
return str(exc), None
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _provider_info():
|
|
133
|
+
"""Which provider auto_provider() picks, or why none is configured. No key values."""
|
|
134
|
+
from .providers import GatewayProvider, auto_provider
|
|
135
|
+
from .providers.detect import _SELECT_ENV
|
|
136
|
+
|
|
137
|
+
env_set = [name for name in PROVIDER_VARS if os.environ.get(name)]
|
|
138
|
+
forced = (os.environ.get('MENDWRIGHT_PROVIDER') or '').strip()
|
|
139
|
+
if forced:
|
|
140
|
+
selected_by = ['MENDWRIGHT_PROVIDER']
|
|
141
|
+
else:
|
|
142
|
+
selected_by = []
|
|
143
|
+
for _, names in _SELECT_ENV:
|
|
144
|
+
hits = [n for n in names if os.environ.get(n)]
|
|
145
|
+
if hits:
|
|
146
|
+
selected_by = hits
|
|
147
|
+
break
|
|
148
|
+
info = {'configured': False, 'name': None, 'model': None, 'selected_by': selected_by,
|
|
149
|
+
'env_set': env_set, 'reason': None, 'gateway': None, 'endpoint': None,
|
|
150
|
+
'disabled': False}
|
|
151
|
+
problem, disabled = _flag_problem('MENDWRIGHT_DISABLE')
|
|
152
|
+
info['disabled'] = bool(disabled)
|
|
153
|
+
try:
|
|
154
|
+
provider = auto_provider()
|
|
155
|
+
except Exception as exc:
|
|
156
|
+
info['reason'] = _scrub(str(exc).strip() or type(exc).__name__)
|
|
157
|
+
return info
|
|
158
|
+
info['configured'] = True
|
|
159
|
+
info['name'] = type(provider).__name__
|
|
160
|
+
if getattr(provider, '_ollama', False) is True:
|
|
161
|
+
info['name'] = 'Ollama (OpenAI-compatible)'
|
|
162
|
+
model = getattr(provider, 'model', None)
|
|
163
|
+
info['model'] = model if isinstance(model, str) and model else None
|
|
164
|
+
if isinstance(provider, GatewayProvider):
|
|
165
|
+
if os.environ.get('MENDWRIGHT_GATEWAY'):
|
|
166
|
+
info['gateway'] = _origin(os.environ['MENDWRIGHT_GATEWAY'])
|
|
167
|
+
else:
|
|
168
|
+
info['endpoint'] = _endpoint(getattr(provider, 'base_url', None))
|
|
169
|
+
return info
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _nearest_existing(folder):
|
|
173
|
+
current = folder
|
|
174
|
+
while current and not os.path.isdir(current):
|
|
175
|
+
parent = os.path.dirname(current)
|
|
176
|
+
if parent == current:
|
|
177
|
+
return None
|
|
178
|
+
current = parent
|
|
179
|
+
return current or None
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _store_info():
|
|
183
|
+
"""The store path open() would use, and whether its folder can be written. Creates
|
|
184
|
+
nothing."""
|
|
185
|
+
env_value = os.environ.get('MENDWRIGHT_STORE')
|
|
186
|
+
raw = env_value or DEFAULT_STORE_PATH
|
|
187
|
+
path = os.path.abspath(os.path.expanduser(raw))
|
|
188
|
+
folder = os.path.dirname(path)
|
|
189
|
+
info = {'path': path, 'source': 'MENDWRIGHT_STORE' if env_value else 'default',
|
|
190
|
+
'exists': os.path.isfile(path), 'folder': folder,
|
|
191
|
+
'folder_exists': os.path.isdir(folder), 'writable': False, 'checked': folder,
|
|
192
|
+
'keys': None, 'journal': os.path.isfile(os.path.join(folder, 'journal.jsonl')),
|
|
193
|
+
'error': None}
|
|
194
|
+
checked = folder if info['folder_exists'] else _nearest_existing(folder)
|
|
195
|
+
info['checked'] = checked
|
|
196
|
+
info['writable'] = bool(checked) and os.access(checked, os.W_OK)
|
|
197
|
+
if info['exists']:
|
|
198
|
+
try:
|
|
199
|
+
with open(path, encoding='utf-8-sig') as handle:
|
|
200
|
+
data = json.loads(handle.read())
|
|
201
|
+
selectors = data.get('selectors') if isinstance(data, dict) else None
|
|
202
|
+
info['keys'] = len(selectors) if isinstance(selectors, (dict, list)) else 0
|
|
203
|
+
except Exception as exc:
|
|
204
|
+
info['error'] = 'could not read it (' + type(exc).__name__ + ')'
|
|
205
|
+
return info
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _env_info():
|
|
209
|
+
out = {}
|
|
210
|
+
for name in sorted(os.environ):
|
|
211
|
+
if not name.upper().startswith('MENDWRIGHT_'):
|
|
212
|
+
continue
|
|
213
|
+
value = os.environ[name]
|
|
214
|
+
if name.upper() == 'MENDWRIGHT_GATEWAY':
|
|
215
|
+
out[name] = _origin(value)
|
|
216
|
+
elif name.upper() in _SHOWN_VALUES:
|
|
217
|
+
out[name] = value
|
|
218
|
+
else:
|
|
219
|
+
out[name] = '(set, value hidden)'
|
|
220
|
+
return out
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _settings_problems():
|
|
224
|
+
from .facade import _env_timeout
|
|
225
|
+
problems = []
|
|
226
|
+
for name in ('MENDWRIGHT_DRY_RUN', 'MENDWRIGHT_DISABLE'):
|
|
227
|
+
problem, _ = _flag_problem(name)
|
|
228
|
+
if problem:
|
|
229
|
+
problems.append(problem)
|
|
230
|
+
try:
|
|
231
|
+
_env_timeout()
|
|
232
|
+
except ValueError as exc:
|
|
233
|
+
problems.append(str(exc))
|
|
234
|
+
return problems
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def collect_doctor():
|
|
238
|
+
"""Everything `mendwright doctor` reports, as a dict (the --json output)."""
|
|
239
|
+
import lxml
|
|
240
|
+
import lxml.etree
|
|
241
|
+
from . import __file__ as package_file
|
|
242
|
+
libxml2 = '.'.join(str(p) for p in getattr(lxml.etree, 'LIBXML_VERSION', ()))
|
|
243
|
+
return {
|
|
244
|
+
'mendwright': {'version': _version(), 'path': os.path.dirname(package_file)},
|
|
245
|
+
'python': {'version': sys.version.split()[0], 'executable': sys.executable},
|
|
246
|
+
'lxml': {'version': getattr(lxml, '__version__', None), 'libxml2': libxml2 or None},
|
|
247
|
+
'browser_libraries': _browser_libraries(),
|
|
248
|
+
'provider': _provider_info(),
|
|
249
|
+
'store': _store_info(),
|
|
250
|
+
'env': _env_info(),
|
|
251
|
+
'problems': [_scrub(p) for p in _settings_problems()],
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _doctor_text(d):
|
|
256
|
+
lines = []
|
|
257
|
+
m, py, lx = d['mendwright'], d['python'], d['lxml']
|
|
258
|
+
lines.append('mendwright ' + m['version'] + ' (' + m['path'] + ')')
|
|
259
|
+
lines.append('Python ' + py['version'] + ' (' + py['executable'] + ')')
|
|
260
|
+
lines.append('lxml ' + str(lx['version'])
|
|
261
|
+
+ (' (libxml2 ' + lx['libxml2'] + ')' if lx['libxml2'] else ''))
|
|
262
|
+
lines.append('')
|
|
263
|
+
|
|
264
|
+
lines.append('Browser libraries (looked up only, nothing imported or launched)')
|
|
265
|
+
for row in d['browser_libraries']:
|
|
266
|
+
state = (row['version'] or 'installed') if row['installed'] else 'not installed'
|
|
267
|
+
lines.append(' ' + row['name'].ljust(11) + ' ' + state)
|
|
268
|
+
if not any(row['installed'] for row in d['browser_libraries']):
|
|
269
|
+
lines.append(' mendwright needs one of these to drive a real page. Install one with: '
|
|
270
|
+
'pip install "mendwright[playwright]"')
|
|
271
|
+
lines.append('')
|
|
272
|
+
|
|
273
|
+
p = d['provider']
|
|
274
|
+
lines.append('Model provider')
|
|
275
|
+
if p['configured']:
|
|
276
|
+
text = p['name'] + (', model ' + p['model'] if p['model'] else '')
|
|
277
|
+
if p['gateway']:
|
|
278
|
+
text += ', gateway ' + p['gateway'] + ' (the model is set on the gateway)'
|
|
279
|
+
lines.append(' ' + text)
|
|
280
|
+
if p.get('endpoint'):
|
|
281
|
+
lines.append(' endpoint: ' + p['endpoint'] + ' (heal requests go here)')
|
|
282
|
+
if p['selected_by']:
|
|
283
|
+
lines.append(' chosen because ' + ' and '.join(p['selected_by'])
|
|
284
|
+
+ (' is' if len(p['selected_by']) == 1 else ' are') + ' set')
|
|
285
|
+
else:
|
|
286
|
+
lines.append(' none configured, so a broken selector is reported but not repaired.')
|
|
287
|
+
lines.append(_wrap('Why: ' + p['reason']))
|
|
288
|
+
lines.append(' `mendwright demo` works without one (it uses a scripted reply).')
|
|
289
|
+
if p['env_set']:
|
|
290
|
+
lines.append(' provider variables set: ' + ', '.join(p['env_set']) + ' (values not shown)')
|
|
291
|
+
if p['disabled']:
|
|
292
|
+
lines.append(' MENDWRIGHT_DISABLE is on, so mendwright.open() heals nothing and calls '
|
|
293
|
+
'no provider. Unset it to heal again.')
|
|
294
|
+
lines.append('')
|
|
295
|
+
|
|
296
|
+
s = d['store']
|
|
297
|
+
lines.append('Store (where mendwright.open() saves heals)')
|
|
298
|
+
origin = ('from MENDWRIGHT_STORE' if s['source'] == 'MENDWRIGHT_STORE'
|
|
299
|
+
else 'default, relative to this folder; set MENDWRIGHT_STORE to change it')
|
|
300
|
+
lines.append(' ' + s['path'])
|
|
301
|
+
lines.append(' (' + origin + ')')
|
|
302
|
+
if s['exists']:
|
|
303
|
+
if s['error']:
|
|
304
|
+
lines.append(' exists, but ' + s['error'] + '. Fix or delete the file.')
|
|
305
|
+
else:
|
|
306
|
+
lines.append(' exists, ' + str(s['keys']) + ' key(s)'
|
|
307
|
+
+ (', journal.jsonl next to it' if s['journal'] else '')
|
|
308
|
+
+ '. See it with: mendwright report')
|
|
309
|
+
elif s['folder_exists']:
|
|
310
|
+
lines.append(' no store yet; it is created the first time mendwright.open() records '
|
|
311
|
+
'a selector')
|
|
312
|
+
else:
|
|
313
|
+
lines.append(' the folder does not exist yet; it is created the first time '
|
|
314
|
+
'mendwright.open() records a selector')
|
|
315
|
+
if s['writable']:
|
|
316
|
+
lines.append(' ' + str(s['checked']) + ' is writable')
|
|
317
|
+
else:
|
|
318
|
+
lines.append(' ' + str(s['checked'] or s['folder']) + ' is NOT writable, so heals '
|
|
319
|
+
'stay in memory. Set MENDWRIGHT_STORE to a writable path.')
|
|
320
|
+
lines.append('')
|
|
321
|
+
|
|
322
|
+
lines.append('Environment')
|
|
323
|
+
if d['env']:
|
|
324
|
+
for name, value in d['env'].items():
|
|
325
|
+
lines.append(' ' + name + '=' + value)
|
|
326
|
+
else:
|
|
327
|
+
lines.append(' no MENDWRIGHT_* variables are set')
|
|
328
|
+
if d['problems']:
|
|
329
|
+
lines.append('')
|
|
330
|
+
lines.append('Problems')
|
|
331
|
+
for problem in d['problems']:
|
|
332
|
+
lines.append(_wrap(problem))
|
|
333
|
+
return lines
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def cmd_doctor(args):
|
|
337
|
+
data = collect_doctor()
|
|
338
|
+
if args.json:
|
|
339
|
+
_out(json.dumps(data, indent=2, sort_keys=True))
|
|
340
|
+
else:
|
|
341
|
+
for line in _doctor_text(data):
|
|
342
|
+
_out(line)
|
|
343
|
+
return 0
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
# --- report ----------------------------------------------------------------------------
|
|
347
|
+
|
|
348
|
+
class ReportError(Exception):
|
|
349
|
+
pass
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def default_store_path():
|
|
353
|
+
return os.path.abspath(os.path.expanduser(os.environ.get('MENDWRIGHT_STORE')
|
|
354
|
+
or DEFAULT_STORE_PATH))
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def _read_store(path):
|
|
358
|
+
try:
|
|
359
|
+
with open(path, encoding='utf-8-sig') as handle:
|
|
360
|
+
text = handle.read()
|
|
361
|
+
except OSError as exc:
|
|
362
|
+
raise ReportError('could not read ' + path + ' (' + (exc.strerror or str(exc))
|
|
363
|
+
+ '). Check the path and its permissions.') from None
|
|
364
|
+
try:
|
|
365
|
+
data = json.loads(text)
|
|
366
|
+
except ValueError as exc:
|
|
367
|
+
raise ReportError(path + ' is not valid JSON (' + str(exc) + '). Fix the file, or '
|
|
368
|
+
'delete it to start an empty store.') from None
|
|
369
|
+
if not isinstance(data, dict):
|
|
370
|
+
data = {}
|
|
371
|
+
selectors = data.get('selectors')
|
|
372
|
+
if data.get('schema') == 2 and isinstance(selectors, dict):
|
|
373
|
+
return data
|
|
374
|
+
if isinstance(selectors, list):
|
|
375
|
+
from .store import migrate_schema1
|
|
376
|
+
return migrate_schema1(data)
|
|
377
|
+
raise ReportError(path + ' is not a mendwright store (expected {"schema": 2, '
|
|
378
|
+
'"selectors": {...}}). Pass the overrides.json file with --store.')
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def _read_journal(path):
|
|
382
|
+
"""The journal's events, oldest first. Unreadable lines are skipped, and a read that
|
|
383
|
+
fails part way (another process holding the file) keeps the events read so far."""
|
|
384
|
+
events = []
|
|
385
|
+
try:
|
|
386
|
+
handle = open(path, encoding='utf-8', errors='replace')
|
|
387
|
+
except OSError:
|
|
388
|
+
return None
|
|
389
|
+
with handle:
|
|
390
|
+
try:
|
|
391
|
+
for line in handle:
|
|
392
|
+
line = line.strip()
|
|
393
|
+
if not line:
|
|
394
|
+
continue
|
|
395
|
+
try:
|
|
396
|
+
event = json.loads(line)
|
|
397
|
+
except ValueError:
|
|
398
|
+
continue
|
|
399
|
+
if isinstance(event, dict) and isinstance(event.get('event'), str):
|
|
400
|
+
events.append(event)
|
|
401
|
+
except OSError:
|
|
402
|
+
pass
|
|
403
|
+
return events
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
_COUNTED = (('requested', 'heal.requested'), ('applied', 'heal.applied'),
|
|
407
|
+
('rejected', 'heal.rejected'), ('quarantined', 'heal.quarantine'))
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def collect_report(path):
|
|
411
|
+
"""The report for the store at ``path`` as a dict (the --format json output).
|
|
412
|
+
Raises ReportError when the store cannot be read."""
|
|
413
|
+
data = _read_store(path)
|
|
414
|
+
keys = []
|
|
415
|
+
for key in sorted(data['selectors']):
|
|
416
|
+
entry = data['selectors'][key]
|
|
417
|
+
if not isinstance(entry, dict):
|
|
418
|
+
continue
|
|
419
|
+
candidates = []
|
|
420
|
+
for c in entry.get('candidates') or []:
|
|
421
|
+
if not isinstance(c, dict) or not c.get('xpath'):
|
|
422
|
+
continue
|
|
423
|
+
row = {'selector': str(c['xpath']), 'source': c.get('source') or 'unknown',
|
|
424
|
+
'status': c.get('status') or 'active',
|
|
425
|
+
'last_success': c.get('lastSucceededAt'), 'added': c.get('addedAt')}
|
|
426
|
+
if c.get('frameXPath'):
|
|
427
|
+
row['frame'] = str(c['frameXPath'])
|
|
428
|
+
candidates.append(row)
|
|
429
|
+
keys.append({'key': key, 'role': entry.get('expectedRole'),
|
|
430
|
+
'destructive': entry.get('destructive') is True,
|
|
431
|
+
'candidates': candidates})
|
|
432
|
+
|
|
433
|
+
journal_path = os.path.join(os.path.dirname(path), 'journal.jsonl')
|
|
434
|
+
events = _read_journal(journal_path)
|
|
435
|
+
journal = {'path': journal_path, 'exists': events is not None,
|
|
436
|
+
'counts': {name: 0 for name, _ in _COUNTED}, 'per_key': {}, 'last_applied': []}
|
|
437
|
+
for event in events or ():
|
|
438
|
+
for name, event_name in _COUNTED:
|
|
439
|
+
if event['event'] == event_name:
|
|
440
|
+
journal['counts'][name] += 1
|
|
441
|
+
per = journal['per_key'].setdefault(str(event.get('key')),
|
|
442
|
+
{n: 0 for n, _ in _COUNTED})
|
|
443
|
+
per[name] += 1
|
|
444
|
+
applied = [e for e in events or () if e['event'] == 'heal.applied']
|
|
445
|
+
for e in reversed(applied[-LAST_HEALS:]):
|
|
446
|
+
old = e.get('old')
|
|
447
|
+
journal['last_applied'].append({
|
|
448
|
+
'when': e.get('ts'), 'key': e.get('key'),
|
|
449
|
+
'old': [str(o) for o in old] if isinstance(old, list) else [],
|
|
450
|
+
'new': e.get('xpath'), 'provider': e.get('provider'), 'model': e.get('model'),
|
|
451
|
+
'latency_ms': e.get('latency_ms')})
|
|
452
|
+
return {'store': path, 'keys': keys, 'journal': journal}
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def _when(ts):
|
|
456
|
+
"""'2026-10-10T09:30:00+00:00' -> '2026-10-10 09:30:00 UTC'."""
|
|
457
|
+
if not isinstance(ts, str) or not ts:
|
|
458
|
+
return 'never'
|
|
459
|
+
text = ts.replace('T', ' ')
|
|
460
|
+
for suffix in ('+00:00', 'Z'):
|
|
461
|
+
if text.endswith(suffix):
|
|
462
|
+
return text[:-len(suffix)] + ' UTC'
|
|
463
|
+
return text
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
def _role_note(k):
|
|
467
|
+
parts = [k['role'] or 'role unknown']
|
|
468
|
+
if k['destructive']:
|
|
469
|
+
parts.append('destructive, never auto-healed')
|
|
470
|
+
return ', '.join(parts)
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def _counts_sentence(counts):
|
|
474
|
+
return ', '.join(str(counts[name]) + ' ' + name for name, _ in _COUNTED)
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
def _who(heal):
|
|
478
|
+
who = heal.get('provider') or 'unknown provider'
|
|
479
|
+
if heal.get('model'):
|
|
480
|
+
who += ', ' + str(heal['model'])
|
|
481
|
+
return who
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def _old_text(heal):
|
|
485
|
+
old = heal['old']
|
|
486
|
+
if not old:
|
|
487
|
+
return '(none)'
|
|
488
|
+
return old[0] + (' (+' + str(len(old) - 1) + ' more)' if len(old) > 1 else '')
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def _report_text(r):
|
|
492
|
+
lines = ['Store: ' + r['store']]
|
|
493
|
+
total = sum(len(k['candidates']) for k in r['keys'])
|
|
494
|
+
lines.append(str(len(r['keys'])) + ' key(s), ' + str(total) + ' selector(s)')
|
|
495
|
+
for k in r['keys']:
|
|
496
|
+
lines.append('')
|
|
497
|
+
lines.append(k['key'] + ' (' + _role_note(k) + ')')
|
|
498
|
+
per = r['journal']['per_key'].get(k['key'])
|
|
499
|
+
for c in k['candidates']:
|
|
500
|
+
frame = ' in frame ' + c['frame'] if c.get('frame') else ''
|
|
501
|
+
lines.append(' ' + c['status'].ljust(11) + ' ' + c['source'].ljust(4) + ' '
|
|
502
|
+
+ c['selector'] + frame + ' (last success: ' + _when(c['last_success'])
|
|
503
|
+
+ ')')
|
|
504
|
+
if per:
|
|
505
|
+
lines.append(' heals: ' + _counts_sentence(per))
|
|
506
|
+
if not r['keys']:
|
|
507
|
+
lines.append('The store has no keys yet.')
|
|
508
|
+
lines.append('')
|
|
509
|
+
j = r['journal']
|
|
510
|
+
if not j['exists']:
|
|
511
|
+
lines.append('No journal.jsonl next to the store, so there are no heal counts.')
|
|
512
|
+
return lines
|
|
513
|
+
lines.append('Heals (journal.jsonl): ' + _counts_sentence(j['counts']))
|
|
514
|
+
if j['last_applied']:
|
|
515
|
+
lines.append('Last applied heals, newest first:')
|
|
516
|
+
for h in j['last_applied']:
|
|
517
|
+
lines.append(' ' + _when(h['when']) + ' ' + str(h['key']) + ' (' + _who(h) + ')')
|
|
518
|
+
lines.append(' ' + _old_text(h) + ' -> ' + str(h['new']))
|
|
519
|
+
return lines
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
def _md_code(text):
|
|
523
|
+
"""text as a Markdown code span that is safe inside a table cell."""
|
|
524
|
+
text = ' '.join(str(text).split()).replace('|', '\\|')
|
|
525
|
+
if '`' in text:
|
|
526
|
+
return '`` ' + text + ' ``'
|
|
527
|
+
return '`' + text + '`'
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
def _md_cell(text):
|
|
531
|
+
return ' '.join(str(text).split()).replace('|', '\\|')
|
|
532
|
+
|
|
533
|
+
|
|
534
|
+
def _report_md(r):
|
|
535
|
+
total = sum(len(k['candidates']) for k in r['keys'])
|
|
536
|
+
lines = ['## mendwright heal report', '',
|
|
537
|
+
'Store: ' + _md_code(r['store']) + ' with ' + str(len(r['keys'])) + ' key(s) and '
|
|
538
|
+
+ str(total) + ' selector(s).', '']
|
|
539
|
+
if r['keys']:
|
|
540
|
+
lines.append('| Key | Selector | Source | Status | Last success |')
|
|
541
|
+
lines.append('| --- | --- | --- | --- | --- |')
|
|
542
|
+
for k in r['keys']:
|
|
543
|
+
first = True
|
|
544
|
+
label = _md_code(k['key']) + ' (' + _md_cell(_role_note(k)) + ')'
|
|
545
|
+
for c in k['candidates'] or [None]:
|
|
546
|
+
cell = label if first else ''
|
|
547
|
+
first = False
|
|
548
|
+
if c is None:
|
|
549
|
+
lines.append('| ' + cell + ' | (none) | | | |')
|
|
550
|
+
continue
|
|
551
|
+
status = c['status'] if c['status'] == 'active' else '**' + c['status'] + '**'
|
|
552
|
+
selector = _md_code(c['selector'])
|
|
553
|
+
if c.get('frame'):
|
|
554
|
+
selector += ' in frame ' + _md_code(c['frame'])
|
|
555
|
+
lines.append('| ' + cell + ' | ' + selector + ' | ' + _md_cell(c['source'])
|
|
556
|
+
+ ' | ' + status + ' | ' + _md_cell(_when(c['last_success'])) + ' |')
|
|
557
|
+
else:
|
|
558
|
+
lines.append('The store has no keys yet.')
|
|
559
|
+
lines.append('')
|
|
560
|
+
j = r['journal']
|
|
561
|
+
if not j['exists']:
|
|
562
|
+
lines.append('No `journal.jsonl` next to the store, so there are no heal counts.')
|
|
563
|
+
return lines
|
|
564
|
+
c = j['counts']
|
|
565
|
+
lines.append('### Heals')
|
|
566
|
+
lines.append('')
|
|
567
|
+
lines.append('| Requested | Applied | Rejected | Quarantined |')
|
|
568
|
+
lines.append('| ---: | ---: | ---: | ---: |')
|
|
569
|
+
lines.append('| ' + ' | '.join(str(c[name]) for name, _ in _COUNTED) + ' |')
|
|
570
|
+
if j['last_applied']:
|
|
571
|
+
lines.append('')
|
|
572
|
+
lines.append('Last applied heals, newest first:')
|
|
573
|
+
lines.append('')
|
|
574
|
+
lines.append('| When | Key | Old | New | Provider |')
|
|
575
|
+
lines.append('| --- | --- | --- | --- | --- |')
|
|
576
|
+
for h in j['last_applied']:
|
|
577
|
+
lines.append('| ' + _md_cell(_when(h['when'])) + ' | ' + _md_code(h['key']) + ' | '
|
|
578
|
+
+ (_md_code(_old_text(h)) if h['old'] else '(none)') + ' | '
|
|
579
|
+
+ _md_code(h['new']) + ' | ' + _md_cell(_who(h)) + ' |')
|
|
580
|
+
return lines
|
|
581
|
+
|
|
582
|
+
|
|
583
|
+
def cmd_report(args):
|
|
584
|
+
path = (os.path.abspath(os.path.expanduser(args.store)) if args.store
|
|
585
|
+
else default_store_path())
|
|
586
|
+
if not os.path.isfile(path):
|
|
587
|
+
if args.store:
|
|
588
|
+
hint = 'Check the path given to --store; the file is created by the first heal'
|
|
589
|
+
else:
|
|
590
|
+
hint = ('Run your tests with mendwright first, or pass --store PATH (the default is '
|
|
591
|
+
'MENDWRIGHT_STORE, else ./.mendwright/overrides.json)')
|
|
592
|
+
_err('mendwright report: no store at ' + path + '. ' + hint + '.')
|
|
593
|
+
return 1
|
|
594
|
+
try:
|
|
595
|
+
report = collect_report(path)
|
|
596
|
+
except ReportError as exc:
|
|
597
|
+
_err('mendwright report: ' + str(exc))
|
|
598
|
+
return 1
|
|
599
|
+
if args.format == 'json':
|
|
600
|
+
_out(json.dumps(report, indent=2))
|
|
601
|
+
else:
|
|
602
|
+
for line in (_report_md(report) if args.format == 'md' else _report_text(report)):
|
|
603
|
+
_out(line)
|
|
604
|
+
return 0
|
|
605
|
+
|
|
606
|
+
|
|
607
|
+
# --- demo ------------------------------------------------------------------------------
|
|
608
|
+
|
|
609
|
+
def cmd_demo(args):
|
|
610
|
+
from .demo import run
|
|
611
|
+
return run(browser=args.browser, keep=args.keep, slow=args.slow, live=args.live)
|
|
612
|
+
|
|
613
|
+
|
|
614
|
+
# --- wiring ----------------------------------------------------------------------------
|
|
615
|
+
|
|
616
|
+
def build_parser():
|
|
617
|
+
parser = argparse.ArgumentParser(
|
|
618
|
+
prog='mendwright',
|
|
619
|
+
description='Self-healing selectors for Playwright and Selenium. When a selector '
|
|
620
|
+
'breaks, a model suggests a new one, mendwright checks it against the '
|
|
621
|
+
'page, and the step is retried.',
|
|
622
|
+
epilog='Run "mendwright <command> --help" for the options of one command.')
|
|
623
|
+
parser.add_argument('-V', '--version', action='store_true',
|
|
624
|
+
help='print the version and exit')
|
|
625
|
+
sub = parser.add_subparsers(dest='command', metavar='<command>')
|
|
626
|
+
|
|
627
|
+
demo = sub.add_parser(
|
|
628
|
+
'demo', help='watch a broken selector get healed (offline, no key needed)',
|
|
629
|
+
description='A redesign breaks two selectors on a checkout page. mendwright heals the '
|
|
630
|
+
'checkbox and refuses to heal the pay button. Runs offline with a stand-in '
|
|
631
|
+
'page and a scripted model reply (no API call); --live asks the model '
|
|
632
|
+
'provider from the environment instead. Writes only to a temp folder, '
|
|
633
|
+
'which is removed afterwards.')
|
|
634
|
+
demo.add_argument('--browser', nargs='?', const='chromium', choices=BROWSERS,
|
|
635
|
+
metavar='{chromium,firefox,webkit}',
|
|
636
|
+
help='run the same story in a real headless Playwright browser '
|
|
637
|
+
'(default: chromium). Needs Playwright and that browser installed.')
|
|
638
|
+
demo.add_argument('--keep', action='store_true',
|
|
639
|
+
help='keep the temp store and journal, and print where they are')
|
|
640
|
+
demo.add_argument('--slow', action='store_true',
|
|
641
|
+
help='pause between lines, for recording a terminal session')
|
|
642
|
+
demo.add_argument('--live', action='store_true',
|
|
643
|
+
help='ask the model provider from the environment (the one '
|
|
644
|
+
'mendwright doctor shows) instead of the scripted reply. Makes a '
|
|
645
|
+
'real API call and sends the demo page to that provider.')
|
|
646
|
+
demo.set_defaults(handler=cmd_demo)
|
|
647
|
+
|
|
648
|
+
doctor = sub.add_parser(
|
|
649
|
+
'doctor', help='check the install, browser libraries, provider and store',
|
|
650
|
+
description='Show versions, which browser libraries are installed (without importing '
|
|
651
|
+
'or launching them), which model provider mendwright.open() would use, '
|
|
652
|
+
'and where heals are stored. Never prints key values. Always exits 0.')
|
|
653
|
+
doctor.add_argument('--json', action='store_true', help='print the result as JSON')
|
|
654
|
+
doctor.set_defaults(handler=cmd_doctor)
|
|
655
|
+
|
|
656
|
+
report = sub.add_parser(
|
|
657
|
+
'report', help='list the selectors and heals in an override store',
|
|
658
|
+
description='List each key in the store with its selectors (source, status, last '
|
|
659
|
+
'success), and the heal counts from journal.jsonl next to it. '
|
|
660
|
+
'--format md pastes into $GITHUB_STEP_SUMMARY.')
|
|
661
|
+
report.add_argument('--store', metavar='PATH',
|
|
662
|
+
help='the override store file (default: MENDWRIGHT_STORE, else '
|
|
663
|
+
'./.mendwright/overrides.json)')
|
|
664
|
+
report.add_argument('--format', choices=REPORT_FORMATS, default='text',
|
|
665
|
+
help='output format (default: text)')
|
|
666
|
+
report.set_defaults(handler=cmd_report)
|
|
667
|
+
return parser
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
def main(argv=None) -> int:
|
|
671
|
+
"""Run the command line with ``argv`` (default: sys.argv[1:]) and return the exit code."""
|
|
672
|
+
parser = build_parser()
|
|
673
|
+
try:
|
|
674
|
+
args = parser.parse_args(argv)
|
|
675
|
+
except SystemExit as exc: # --help, or a usage error argparse already printed
|
|
676
|
+
code = exc.code
|
|
677
|
+
return code if isinstance(code, int) else (0 if code is None else 2)
|
|
678
|
+
if args.version:
|
|
679
|
+
_out('mendwright ' + _version())
|
|
680
|
+
return 0
|
|
681
|
+
handler = getattr(args, 'handler', None)
|
|
682
|
+
if handler is None:
|
|
683
|
+
parser.print_help()
|
|
684
|
+
return 0
|
|
685
|
+
try:
|
|
686
|
+
return handler(args)
|
|
687
|
+
except KeyboardInterrupt:
|
|
688
|
+
_err('mendwright: interrupted.')
|
|
689
|
+
return 130
|