webtap-tool 0.7.1__py3-none-any.whl → 0.8.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of webtap-tool might be problematic. Click here for more details.
- webtap/__init__.py +4 -0
- webtap/api.py +50 -57
- webtap/app.py +5 -0
- webtap/cdp/session.py +77 -25
- webtap/commands/TIPS.md +125 -22
- webtap/commands/_builders.py +7 -1
- webtap/commands/_code_generation.py +110 -0
- webtap/commands/body.py +9 -5
- webtap/commands/connection.py +21 -0
- webtap/commands/javascript.py +13 -25
- webtap/commands/navigation.py +5 -0
- webtap/commands/quicktype.py +268 -0
- webtap/commands/to_model.py +23 -75
- webtap/services/body.py +209 -24
- webtap/services/dom.py +19 -12
- webtap/services/fetch.py +19 -0
- webtap/services/main.py +192 -0
- webtap/services/setup/extension.py +1 -1
- webtap/services/state_snapshot.py +88 -0
- {webtap_tool-0.7.1.dist-info → webtap_tool-0.8.1.dist-info}/METADATA +1 -1
- {webtap_tool-0.7.1.dist-info → webtap_tool-0.8.1.dist-info}/RECORD +23 -20
- {webtap_tool-0.7.1.dist-info → webtap_tool-0.8.1.dist-info}/WHEEL +0 -0
- {webtap_tool-0.7.1.dist-info → webtap_tool-0.8.1.dist-info}/entry_points.txt +0 -0
webtap/__init__.py
CHANGED
|
@@ -9,10 +9,14 @@ PUBLIC API:
|
|
|
9
9
|
- main: Entry point function for CLI
|
|
10
10
|
"""
|
|
11
11
|
|
|
12
|
+
import atexit
|
|
12
13
|
import sys
|
|
13
14
|
|
|
14
15
|
from webtap.app import app
|
|
15
16
|
|
|
17
|
+
# Register cleanup on exit to shutdown DB thread
|
|
18
|
+
atexit.register(lambda: app.state.cleanup() if hasattr(app, "state") and app.state else None)
|
|
19
|
+
|
|
16
20
|
|
|
17
21
|
def main():
|
|
18
22
|
"""Entry point for the WebTap REPL.
|
webtap/api.py
CHANGED
|
@@ -119,9 +119,6 @@ async def connect(request: ConnectRequest) -> Dict[str, Any]:
|
|
|
119
119
|
# Wrap blocking CDP calls (connect + enable domains) in thread
|
|
120
120
|
result = await asyncio.to_thread(app_state.service.connect_to_page, page_id=request.page_id)
|
|
121
121
|
|
|
122
|
-
# Broadcast state change
|
|
123
|
-
await broadcast_state()
|
|
124
|
-
|
|
125
122
|
return result
|
|
126
123
|
|
|
127
124
|
|
|
@@ -134,9 +131,6 @@ async def disconnect() -> Dict[str, Any]:
|
|
|
134
131
|
# Wrap blocking CDP calls (fetch.disable + disconnect) in thread
|
|
135
132
|
result = await asyncio.to_thread(app_state.service.disconnect)
|
|
136
133
|
|
|
137
|
-
# Broadcast state change
|
|
138
|
-
await broadcast_state()
|
|
139
|
-
|
|
140
134
|
return result
|
|
141
135
|
|
|
142
136
|
|
|
@@ -146,7 +140,10 @@ async def clear_events() -> Dict[str, Any]:
|
|
|
146
140
|
if not app_state:
|
|
147
141
|
return {"error": "WebTap not initialized"}
|
|
148
142
|
|
|
149
|
-
|
|
143
|
+
# Wrap blocking DB operation in thread
|
|
144
|
+
result = await asyncio.to_thread(app_state.service.clear_events)
|
|
145
|
+
|
|
146
|
+
return result
|
|
150
147
|
|
|
151
148
|
|
|
152
149
|
@api.post("/fetch")
|
|
@@ -164,7 +161,7 @@ async def set_fetch_interception(request: FetchRequest) -> Dict[str, Any]:
|
|
|
164
161
|
result = await asyncio.to_thread(app_state.service.fetch.disable)
|
|
165
162
|
|
|
166
163
|
# Broadcast state change
|
|
167
|
-
|
|
164
|
+
app_state.service._trigger_broadcast()
|
|
168
165
|
|
|
169
166
|
return result
|
|
170
167
|
|
|
@@ -200,7 +197,7 @@ async def toggle_filter_category(category: str) -> Dict[str, Any]:
|
|
|
200
197
|
fm.save()
|
|
201
198
|
|
|
202
199
|
# Broadcast state change
|
|
203
|
-
|
|
200
|
+
app_state.service._trigger_broadcast()
|
|
204
201
|
|
|
205
202
|
return {"category": category, "enabled": enabled, "total_enabled": len(fm.enabled_categories)}
|
|
206
203
|
|
|
@@ -216,7 +213,7 @@ async def enable_all_filters() -> Dict[str, Any]:
|
|
|
216
213
|
fm.save()
|
|
217
214
|
|
|
218
215
|
# Broadcast state change
|
|
219
|
-
|
|
216
|
+
app_state.service._trigger_broadcast()
|
|
220
217
|
|
|
221
218
|
return {"enabled": list(fm.enabled_categories), "total": len(fm.enabled_categories)}
|
|
222
219
|
|
|
@@ -232,7 +229,7 @@ async def disable_all_filters() -> Dict[str, Any]:
|
|
|
232
229
|
fm.save()
|
|
233
230
|
|
|
234
231
|
# Broadcast state change
|
|
235
|
-
|
|
232
|
+
app_state.service._trigger_broadcast()
|
|
236
233
|
|
|
237
234
|
return {"enabled": [], "total": 0}
|
|
238
235
|
|
|
@@ -249,9 +246,6 @@ async def start_inspect() -> Dict[str, Any]:
|
|
|
249
246
|
# Wrap blocking CDP calls (DOM.enable, CSS.enable, Overlay.enable, setInspectMode) in thread
|
|
250
247
|
result = await asyncio.to_thread(app_state.service.dom.start_inspect)
|
|
251
248
|
|
|
252
|
-
# Broadcast state change
|
|
253
|
-
await broadcast_state()
|
|
254
|
-
|
|
255
249
|
return result
|
|
256
250
|
|
|
257
251
|
|
|
@@ -264,9 +258,6 @@ async def stop_inspect() -> Dict[str, Any]:
|
|
|
264
258
|
# Wrap blocking CDP call (Overlay.setInspectMode) in thread
|
|
265
259
|
result = await asyncio.to_thread(app_state.service.dom.stop_inspect)
|
|
266
260
|
|
|
267
|
-
# Broadcast state change
|
|
268
|
-
await broadcast_state()
|
|
269
|
-
|
|
270
261
|
return result
|
|
271
262
|
|
|
272
263
|
|
|
@@ -278,9 +269,6 @@ async def clear_selections() -> Dict[str, Any]:
|
|
|
278
269
|
|
|
279
270
|
app_state.service.dom.clear_selections()
|
|
280
271
|
|
|
281
|
-
# Broadcast state change
|
|
282
|
-
await broadcast_state()
|
|
283
|
-
|
|
284
272
|
return {"success": True, "selections": {}}
|
|
285
273
|
|
|
286
274
|
|
|
@@ -293,7 +281,7 @@ async def dismiss_error() -> Dict[str, Any]:
|
|
|
293
281
|
app_state.error_state = None
|
|
294
282
|
|
|
295
283
|
# Broadcast state change
|
|
296
|
-
|
|
284
|
+
app_state.service._trigger_broadcast()
|
|
297
285
|
|
|
298
286
|
return {"success": True}
|
|
299
287
|
|
|
@@ -363,11 +351,11 @@ async def stream_events():
|
|
|
363
351
|
def get_full_state() -> Dict[str, Any]:
|
|
364
352
|
"""Get complete WebTap state for broadcasting.
|
|
365
353
|
|
|
366
|
-
|
|
367
|
-
|
|
354
|
+
Thread-safe, zero-lock reads from immutable snapshot.
|
|
355
|
+
No blocking I/O - returns cached snapshot immediately.
|
|
368
356
|
|
|
369
357
|
Returns:
|
|
370
|
-
Dictionary with all state information
|
|
358
|
+
Dictionary with all state information for SSE clients
|
|
371
359
|
"""
|
|
372
360
|
if not app_state:
|
|
373
361
|
return {
|
|
@@ -375,40 +363,35 @@ def get_full_state() -> Dict[str, Any]:
|
|
|
375
363
|
"events": {"total": 0},
|
|
376
364
|
"fetch": {"enabled": False, "paused_count": 0},
|
|
377
365
|
"filters": {"enabled": [], "disabled": []},
|
|
378
|
-
"browser": {"inspect_active": False, "selections": {}, "prompt": ""},
|
|
366
|
+
"browser": {"inspect_active": False, "selections": {}, "prompt": "", "pending_count": 0},
|
|
379
367
|
"error": None,
|
|
380
368
|
}
|
|
381
369
|
|
|
382
|
-
# Get
|
|
383
|
-
|
|
384
|
-
page_info = app_state.cdp.page_info or {}
|
|
385
|
-
|
|
386
|
-
# Get event counts
|
|
387
|
-
event_count = app_state.service.event_count
|
|
388
|
-
|
|
389
|
-
# Get fetch status
|
|
390
|
-
fetch_enabled = app_state.service.fetch.enabled
|
|
391
|
-
paused_count = app_state.service.fetch.paused_count if fetch_enabled else 0
|
|
392
|
-
|
|
393
|
-
# Get filter status
|
|
394
|
-
fm = app_state.service.filters
|
|
395
|
-
filter_categories = list(fm.filters.keys())
|
|
396
|
-
enabled_filters = list(fm.enabled_categories)
|
|
397
|
-
disabled_filters = [cat for cat in filter_categories if cat not in enabled_filters]
|
|
398
|
-
|
|
399
|
-
# Get browser/DOM state (includes pending_count for progress indicator)
|
|
400
|
-
browser_state = app_state.service.dom.get_state()
|
|
370
|
+
# Get immutable snapshot (NO LOCKS NEEDED - inherently thread-safe)
|
|
371
|
+
snapshot = app_state.service.get_state_snapshot()
|
|
401
372
|
|
|
373
|
+
# Convert snapshot to frontend format
|
|
402
374
|
return {
|
|
403
|
-
"connected": connected,
|
|
404
|
-
"page": {
|
|
405
|
-
|
|
375
|
+
"connected": snapshot.connected,
|
|
376
|
+
"page": {
|
|
377
|
+
"id": snapshot.page_id,
|
|
378
|
+
"title": snapshot.page_title,
|
|
379
|
+
"url": snapshot.page_url,
|
|
380
|
+
}
|
|
381
|
+
if snapshot.connected
|
|
382
|
+
else None,
|
|
383
|
+
"events": {"total": snapshot.event_count},
|
|
384
|
+
"fetch": {"enabled": snapshot.fetch_enabled, "paused_count": snapshot.paused_count},
|
|
385
|
+
"filters": {"enabled": list(snapshot.enabled_filters), "disabled": list(snapshot.disabled_filters)},
|
|
386
|
+
"browser": {
|
|
387
|
+
"inspect_active": snapshot.inspect_active,
|
|
388
|
+
"selections": snapshot.selections,
|
|
389
|
+
"prompt": snapshot.prompt,
|
|
390
|
+
"pending_count": snapshot.pending_count,
|
|
391
|
+
},
|
|
392
|
+
"error": {"message": snapshot.error_message, "timestamp": snapshot.error_timestamp}
|
|
393
|
+
if snapshot.error_message
|
|
406
394
|
else None,
|
|
407
|
-
"events": {"total": event_count},
|
|
408
|
-
"fetch": {"enabled": fetch_enabled, "paused_count": paused_count},
|
|
409
|
-
"filters": {"enabled": enabled_filters, "disabled": disabled_filters},
|
|
410
|
-
"browser": browser_state, # Contains inspect_active, selections, prompt, pending_count
|
|
411
|
-
"error": app_state.error_state, # Current error or None
|
|
412
395
|
}
|
|
413
396
|
|
|
414
397
|
|
|
@@ -429,7 +412,14 @@ async def broadcast_state():
|
|
|
429
412
|
try:
|
|
430
413
|
queue.put_nowait(state)
|
|
431
414
|
except asyncio.QueueFull:
|
|
432
|
-
|
|
415
|
+
# Client is falling behind - discard oldest state and retry with latest
|
|
416
|
+
logger.warning(f"SSE client queue full ({queue.qsize()}/{queue.maxsize}), discarding oldest state")
|
|
417
|
+
try:
|
|
418
|
+
queue.get_nowait() # Discard oldest
|
|
419
|
+
queue.put_nowait(state) # Retry with latest
|
|
420
|
+
except Exception as retry_err:
|
|
421
|
+
logger.debug(f"Failed to recover full queue: {retry_err}")
|
|
422
|
+
dead_queues.add(queue)
|
|
433
423
|
except Exception as e:
|
|
434
424
|
logger.debug(f"Failed to broadcast to client: {e}")
|
|
435
425
|
dead_queues.add(queue)
|
|
@@ -470,7 +460,9 @@ async def broadcast_processor():
|
|
|
470
460
|
async with _sse_clients_lock:
|
|
471
461
|
for queue in list(_sse_clients):
|
|
472
462
|
try:
|
|
473
|
-
|
|
463
|
+
queue.put_nowait(None) # Non-blocking shutdown signal
|
|
464
|
+
except asyncio.QueueFull:
|
|
465
|
+
pass # Client is hung, skip
|
|
474
466
|
except Exception:
|
|
475
467
|
pass
|
|
476
468
|
_sse_clients.clear()
|
|
@@ -521,11 +513,12 @@ def start_api_server(state, host: str = "127.0.0.1", port: int = 8765) -> thread
|
|
|
521
513
|
logger.error("Broadcast queue initialization timed out")
|
|
522
514
|
return thread
|
|
523
515
|
|
|
524
|
-
# Wire queue to
|
|
516
|
+
# Wire queue to service and CDP session after event loop starts
|
|
517
|
+
# Note: DOMService uses callback to service._trigger_broadcast instead of direct queue access
|
|
525
518
|
if _broadcast_queue and app_state:
|
|
526
|
-
app_state.service.
|
|
519
|
+
app_state.service.set_broadcast_queue(_broadcast_queue)
|
|
527
520
|
app_state.cdp.set_broadcast_queue(_broadcast_queue)
|
|
528
|
-
logger.info("Broadcast queue wired to
|
|
521
|
+
logger.info("Broadcast queue wired to WebTapService and CDPSession")
|
|
529
522
|
|
|
530
523
|
logger.info(f"API server started on http://{host}:{port}")
|
|
531
524
|
return thread
|
webtap/app.py
CHANGED
|
@@ -57,6 +57,10 @@ class WebTapState:
|
|
|
57
57
|
# Give server 1.5s to close SSE connections and shutdown gracefully
|
|
58
58
|
self.api_thread.join(timeout=1.5)
|
|
59
59
|
|
|
60
|
+
# Shutdown DB thread (this is the only place where DB thread should stop)
|
|
61
|
+
if hasattr(self, "cdp") and self.cdp:
|
|
62
|
+
self.cdp.cleanup()
|
|
63
|
+
|
|
60
64
|
|
|
61
65
|
# Must be created before command imports for decorator registration
|
|
62
66
|
app = App(
|
|
@@ -90,6 +94,7 @@ else:
|
|
|
90
94
|
from webtap.commands import fetch # noqa: E402, F401
|
|
91
95
|
from webtap.commands import body # noqa: E402, F401
|
|
92
96
|
from webtap.commands import to_model # noqa: E402, F401
|
|
97
|
+
from webtap.commands import quicktype # noqa: E402, F401
|
|
93
98
|
from webtap.commands import selections # noqa: E402, F401
|
|
94
99
|
from webtap.commands import server # noqa: E402, F401
|
|
95
100
|
from webtap.commands import setup # noqa: E402, F401
|
webtap/cdp/session.py
CHANGED
|
@@ -78,8 +78,9 @@ class CDPSession:
|
|
|
78
78
|
|
|
79
79
|
# Broadcast queue for SSE state updates (set by API server)
|
|
80
80
|
self._broadcast_queue: "Any | None" = None
|
|
81
|
-
|
|
82
|
-
|
|
81
|
+
|
|
82
|
+
# Disconnect callback for service-level cleanup
|
|
83
|
+
self._disconnect_callback: "Any | None" = None
|
|
83
84
|
|
|
84
85
|
def _db_worker(self) -> None:
|
|
85
86
|
"""Dedicated thread for all database operations.
|
|
@@ -218,7 +219,7 @@ class CDPSession:
|
|
|
218
219
|
kwargs={
|
|
219
220
|
"ping_interval": 30, # Ping every 30s
|
|
220
221
|
"ping_timeout": 20, # Wait 20s for pong (increased from 10s for heavy CDP load)
|
|
221
|
-
|
|
222
|
+
# No auto-reconnect - make disconnects explicit
|
|
222
223
|
"skip_utf8_validation": True, # Faster
|
|
223
224
|
},
|
|
224
225
|
)
|
|
@@ -231,24 +232,46 @@ class CDPSession:
|
|
|
231
232
|
raise TimeoutError("Failed to connect to Chrome")
|
|
232
233
|
|
|
233
234
|
def disconnect(self) -> None:
|
|
234
|
-
"""Disconnect WebSocket and
|
|
235
|
-
|
|
236
|
-
|
|
235
|
+
"""Disconnect WebSocket while preserving events and DB thread.
|
|
236
|
+
|
|
237
|
+
Events and DB thread persist across connection cycles.
|
|
238
|
+
Use cleanup() on app exit to shutdown DB thread.
|
|
239
|
+
"""
|
|
240
|
+
# Atomically clear ws_app to signal manual disconnect
|
|
241
|
+
# This prevents _on_close from triggering service callback
|
|
242
|
+
with self._lock:
|
|
243
|
+
ws_app = self.ws_app
|
|
237
244
|
self.ws_app = None
|
|
238
245
|
|
|
246
|
+
if ws_app:
|
|
247
|
+
ws_app.close()
|
|
248
|
+
|
|
239
249
|
if self.ws_thread and self.ws_thread.is_alive():
|
|
240
250
|
self.ws_thread.join(timeout=2)
|
|
241
251
|
self.ws_thread = None
|
|
242
252
|
|
|
253
|
+
# Keep DB thread running - events preserved for reconnection
|
|
254
|
+
# DB cleanup happens in cleanup() on app exit only
|
|
255
|
+
|
|
256
|
+
self.connected.clear()
|
|
257
|
+
self.page_info = None
|
|
258
|
+
|
|
259
|
+
def cleanup(self) -> None:
|
|
260
|
+
"""Shutdown DB thread and disconnect (call on app exit only).
|
|
261
|
+
|
|
262
|
+
This is the only place where DB thread should be stopped.
|
|
263
|
+
Events are lost when DB thread stops (in-memory database).
|
|
264
|
+
"""
|
|
265
|
+
# Disconnect WebSocket if connected
|
|
266
|
+
if self.ws_app:
|
|
267
|
+
self.disconnect()
|
|
268
|
+
|
|
243
269
|
# Shutdown database thread
|
|
244
270
|
self._db_running = False
|
|
245
271
|
self._db_work_queue.put(None) # Signal shutdown
|
|
246
272
|
if self._db_thread.is_alive():
|
|
247
273
|
self._db_thread.join(timeout=2)
|
|
248
274
|
|
|
249
|
-
self.connected.clear()
|
|
250
|
-
self.page_info = None
|
|
251
|
-
|
|
252
275
|
def send(self, method: str, params: dict | None = None) -> Future:
|
|
253
276
|
"""Send CDP command asynchronously.
|
|
254
277
|
|
|
@@ -351,15 +374,39 @@ class CDPSession:
|
|
|
351
374
|
|
|
352
375
|
def _on_close(self, ws, code, reason):
|
|
353
376
|
"""Handle WebSocket closure and cleanup."""
|
|
354
|
-
logger.info(f"WebSocket closed: {code} {reason}")
|
|
377
|
+
logger.info(f"WebSocket closed: code={code} reason={reason}")
|
|
378
|
+
|
|
379
|
+
# Mark as disconnected
|
|
380
|
+
was_connected = self.connected.is_set()
|
|
355
381
|
self.connected.clear()
|
|
356
382
|
|
|
357
|
-
# Fail pending commands
|
|
383
|
+
# Fail pending commands and check if this is unexpected disconnect
|
|
384
|
+
is_unexpected = False
|
|
358
385
|
with self._lock:
|
|
359
386
|
for future in self._pending.values():
|
|
360
|
-
future.set_exception(RuntimeError("Connection closed"))
|
|
387
|
+
future.set_exception(RuntimeError(f"Connection closed: {reason or 'Unknown'}"))
|
|
361
388
|
self._pending.clear()
|
|
362
389
|
|
|
390
|
+
# Unexpected disconnect: was connected and ws_app still set (not manual disconnect)
|
|
391
|
+
is_unexpected = was_connected and self.ws_app is not None
|
|
392
|
+
|
|
393
|
+
# Clear state to allow reconnection (DB thread and events preserved)
|
|
394
|
+
self.ws_app = None
|
|
395
|
+
self.page_info = None
|
|
396
|
+
|
|
397
|
+
# Trigger service-level cleanup if this was unexpected
|
|
398
|
+
if is_unexpected and self._disconnect_callback:
|
|
399
|
+
try:
|
|
400
|
+
# Call in background to avoid blocking WebSocket thread
|
|
401
|
+
threading.Thread(
|
|
402
|
+
target=self._disconnect_callback, args=(code, reason), daemon=True, name="cdp-disconnect-handler"
|
|
403
|
+
).start()
|
|
404
|
+
except Exception as e:
|
|
405
|
+
logger.error(f"Error calling disconnect callback: {e}")
|
|
406
|
+
|
|
407
|
+
# Trigger SSE broadcast immediately
|
|
408
|
+
self._trigger_state_broadcast()
|
|
409
|
+
|
|
363
410
|
def _extract_paths(self, obj, parent_key=""):
|
|
364
411
|
"""Extract all JSON paths from nested dict structure.
|
|
365
412
|
|
|
@@ -466,6 +513,18 @@ class CDPSession:
|
|
|
466
513
|
"""
|
|
467
514
|
return self.connected.is_set()
|
|
468
515
|
|
|
516
|
+
def set_disconnect_callback(self, callback) -> None:
|
|
517
|
+
"""Register callback for unexpected disconnect events.
|
|
518
|
+
|
|
519
|
+
Called when WebSocket closes externally (tab close, crash, etc).
|
|
520
|
+
NOT called on manual disconnect() to avoid duplicate cleanup.
|
|
521
|
+
|
|
522
|
+
Args:
|
|
523
|
+
callback: Function called with (code: int, reason: str)
|
|
524
|
+
"""
|
|
525
|
+
self._disconnect_callback = callback
|
|
526
|
+
logger.debug("Disconnect callback registered")
|
|
527
|
+
|
|
469
528
|
def register_event_callback(self, method: str, callback) -> None:
|
|
470
529
|
"""Register callback for specific CDP event.
|
|
471
530
|
|
|
@@ -532,21 +591,14 @@ class CDPSession:
|
|
|
532
591
|
logger.debug("Broadcast queue set on CDPSession")
|
|
533
592
|
|
|
534
593
|
def _trigger_state_broadcast(self) -> None:
|
|
535
|
-
"""Trigger SSE broadcast
|
|
594
|
+
"""Trigger SSE broadcast immediately.
|
|
536
595
|
|
|
537
|
-
Called after CDP events are stored.
|
|
538
|
-
to avoid overwhelming SSE clients during heavy network activity.
|
|
596
|
+
Called after CDP events are stored. Queue naturally buffers rapid-fire events.
|
|
539
597
|
"""
|
|
540
598
|
if not self._broadcast_queue:
|
|
541
599
|
return
|
|
542
600
|
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
self._last_broadcast_time = now
|
|
548
|
-
try:
|
|
549
|
-
self._broadcast_queue.put_nowait({"type": "cdp_event"})
|
|
550
|
-
logger.debug("State broadcast triggered")
|
|
551
|
-
except Exception as e:
|
|
552
|
-
logger.debug(f"Failed to queue broadcast: {e}")
|
|
601
|
+
try:
|
|
602
|
+
self._broadcast_queue.put_nowait({"type": "cdp_event"})
|
|
603
|
+
except Exception as e:
|
|
604
|
+
logger.debug(f"Failed to queue broadcast: {e}")
|
webtap/commands/TIPS.md
CHANGED
|
@@ -12,13 +12,21 @@ All commands have these pre-imported (no imports needed!):
|
|
|
12
12
|
## Commands
|
|
13
13
|
|
|
14
14
|
### body
|
|
15
|
-
Fetch and analyze HTTP response bodies with Python expressions.
|
|
15
|
+
Fetch and analyze HTTP request or response bodies with Python expressions. Automatically detects event type.
|
|
16
16
|
|
|
17
17
|
#### Examples
|
|
18
18
|
```python
|
|
19
|
-
|
|
20
|
-
body(123
|
|
19
|
+
# Response bodies
|
|
20
|
+
body(123) # Get response body
|
|
21
|
+
body(123, "json.loads(body)") # Parse JSON response
|
|
21
22
|
body(123, "bs4(body, 'html.parser').find('title').text") # HTML title
|
|
23
|
+
|
|
24
|
+
# Request bodies (POST/PUT/PATCH)
|
|
25
|
+
body(456) # Get request POST data
|
|
26
|
+
body(456, "json.loads(body)") # Parse JSON request body
|
|
27
|
+
body(456, "json.loads(body)['customerId']") # Extract request field
|
|
28
|
+
|
|
29
|
+
# Analysis
|
|
22
30
|
body(123, "jwt.decode(body, options={'verify_signature': False})") # Decode JWT
|
|
23
31
|
body(123, "re.findall(r'/api/[^\"\\s]+', body)[:10]") # Find API endpoints
|
|
24
32
|
body(123, "httpx.get(json.loads(body)['next_url']).json()") # Chain requests
|
|
@@ -26,7 +34,10 @@ body(123, "msgpack.unpackb(body)") # Binary formats
|
|
|
26
34
|
```
|
|
27
35
|
|
|
28
36
|
#### Tips
|
|
37
|
+
- **Auto-detect type:** Command automatically detects request vs response events
|
|
38
|
+
- **Find request events:** `events({"method": "Network.requestWillBeSent", "url": "*WsJobCard/Post*"})` - POST requests
|
|
29
39
|
- **Generate models:** `to_model({id}, "models/model.py", "Model")` - create Pydantic models from JSON
|
|
40
|
+
- **Generate types:** `quicktype({id}, "types.ts", "Type")` - TypeScript/other languages
|
|
30
41
|
- **Chain requests:** `body({id}, "httpx.get(json.loads(body)['next_url']).text[:100]")`
|
|
31
42
|
- **Parse XML:** `body({id}, "ElementTree.fromstring(body).find('.//title').text")`
|
|
32
43
|
- **Extract forms:** `body({id}, "[f['action'] for f in bs4(body, 'html.parser').find_all('form')]")`
|
|
@@ -34,22 +45,62 @@ body(123, "msgpack.unpackb(body)") # Binary formats
|
|
|
34
45
|
- **Find related:** `events({'requestId': request_id})` - related events
|
|
35
46
|
|
|
36
47
|
### to_model
|
|
37
|
-
Generate Pydantic v2 models from
|
|
48
|
+
Generate Pydantic v2 models from request or response bodies.
|
|
38
49
|
|
|
39
50
|
#### Examples
|
|
40
51
|
```python
|
|
41
|
-
|
|
42
|
-
to_model(123, "models/
|
|
43
|
-
to_model(123, "/
|
|
52
|
+
# Response bodies
|
|
53
|
+
to_model(123, "models/product.py", "Product") # Full response
|
|
54
|
+
to_model(123, "models/customers/group.py", "CustomerGroup", json_path="data[0]") # Extract nested
|
|
55
|
+
|
|
56
|
+
# Request bodies (POST/PUT/PATCH)
|
|
57
|
+
to_model(172, "models/form.py", "JobCardForm", expr="dict(urllib.parse.parse_qsl(body))") # Form data
|
|
58
|
+
to_model(180, "models/request.py", "CreateOrder") # JSON POST body
|
|
59
|
+
|
|
60
|
+
# Advanced transformations
|
|
61
|
+
to_model(123, "models/clean.py", "Clean", expr="{k: v for k, v in json.loads(body).items() if k != 'meta'}")
|
|
62
|
+
to_model(123, "models/merged.py", "Merged", expr="{**json.loads(body), 'url': event['params']['response']['url']}")
|
|
44
63
|
```
|
|
45
64
|
|
|
46
65
|
#### Tips
|
|
47
|
-
- **Check structure
|
|
48
|
-
- **
|
|
49
|
-
- **
|
|
50
|
-
- **
|
|
51
|
-
- **
|
|
52
|
-
- **
|
|
66
|
+
- **Check structure:** `body({id})` - preview body before generating
|
|
67
|
+
- **Find requests:** `events({"method": "Network.requestWillBeSent", "url": "*api/orders*"})` - locate POST events
|
|
68
|
+
- **Form data:** `expr="dict(urllib.parse.parse_qsl(body))"` for application/x-www-form-urlencoded
|
|
69
|
+
- **Nested extraction:** `json_path="data[0]"` for JSON with wrapper objects
|
|
70
|
+
- **Custom transforms:** `expr` has `body` (str) and `event` (dict) variables available
|
|
71
|
+
- **Organization:** Paths like `"models/customers/group.py"` create directory structure automatically
|
|
72
|
+
- **Field mapping:** Add `Field(alias="...")` after generation for API field names
|
|
73
|
+
|
|
74
|
+
### quicktype
|
|
75
|
+
Generate types from request or response bodies. Supports TypeScript, Go, Rust, Python, and 10+ other languages.
|
|
76
|
+
|
|
77
|
+
#### Examples
|
|
78
|
+
```python
|
|
79
|
+
# Response bodies
|
|
80
|
+
quicktype(123, "types/User.ts", "User") # TypeScript
|
|
81
|
+
quicktype(123, "api.go", "ApiResponse") # Go struct
|
|
82
|
+
quicktype(123, "schema.json", "Schema") # JSON Schema
|
|
83
|
+
quicktype(123, "types.ts", "User", json_path="data[0]") # Extract nested
|
|
84
|
+
|
|
85
|
+
# Request bodies (POST/PUT/PATCH)
|
|
86
|
+
quicktype(172, "types/JobCard.ts", "JobCardForm", expr="dict(urllib.parse.parse_qsl(body))") # Form data
|
|
87
|
+
quicktype(180, "types/CreateOrder.ts", "CreateOrderRequest") # JSON POST body
|
|
88
|
+
|
|
89
|
+
# Advanced options
|
|
90
|
+
quicktype(123, "types.ts", "User", options={"readonly": True}) # TypeScript readonly
|
|
91
|
+
quicktype(123, "types.ts", "Clean", expr="{k: v for k, v in json.loads(body).items() if k != 'meta'}")
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
#### Tips
|
|
95
|
+
- **Check structure:** `body({id})` - preview body before generating
|
|
96
|
+
- **Find requests:** `events({"method": "Network.requestWillBeSent", "url": "*api*"})` - locate POST events
|
|
97
|
+
- **Form data:** `expr="dict(urllib.parse.parse_qsl(body))"` for application/x-www-form-urlencoded
|
|
98
|
+
- **Nested extraction:** `json_path="data[0]"` for JSON with wrapper objects
|
|
99
|
+
- **Languages:** .ts/.go/.rs/.java/.kt/.swift/.cs/.cpp/.dart/.rb/.json extensions set language
|
|
100
|
+
- **Options:** Dict keys map to CLI flags: `{"readonly": True}` → `--readonly`, `{"nice_property_names": True}` → `--nice-property-names`. See `quicktype --help` for language-specific flags
|
|
101
|
+
- **Common options:** TypeScript: `{"readonly": True, "prefer_types": True}`, Go: `{"omit_empty": True}`, Python: `{"pydantic_base_model": True}`
|
|
102
|
+
- **Install:** `npm install -g quicktype` if command not found
|
|
103
|
+
- **Pydantic models:** Use `to_model({id}, "models/model.py", "Model")` for Pydantic v2 instead
|
|
53
104
|
|
|
54
105
|
### inspect
|
|
55
106
|
Inspect CDP events with full Python debugging.
|
|
@@ -123,25 +174,54 @@ events({"level": "error"}) # Console errors
|
|
|
123
174
|
- **Decode data:** `inspect({id}, "base64.b64decode(data.get('params', {}).get('body', ''))")`
|
|
124
175
|
|
|
125
176
|
### js
|
|
126
|
-
Execute JavaScript in the browser
|
|
177
|
+
Execute JavaScript in the browser. Uses fresh scope by default to avoid redeclaration errors.
|
|
178
|
+
|
|
179
|
+
#### Scope Behavior
|
|
180
|
+
**Default (fresh scope)** - Each call runs in isolation:
|
|
181
|
+
```python
|
|
182
|
+
js("const x = 1") # ✓ x isolated
|
|
183
|
+
js("const x = 2") # ✓ No error, fresh scope
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
**Persistent scope** - Variables survive across calls:
|
|
187
|
+
```python
|
|
188
|
+
js("var data = {count: 0}", persist=True) # data persists
|
|
189
|
+
js("data.count++", persist=True) # Modifies data
|
|
190
|
+
js("data.count", persist=True) # Returns 1
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
**With browser element** - Always fresh scope:
|
|
194
|
+
```python
|
|
195
|
+
js("element.offsetWidth", selection=1) # Use element #1
|
|
196
|
+
js("element.classList", selection=2) # Use element #2
|
|
197
|
+
```
|
|
127
198
|
|
|
128
199
|
#### Examples
|
|
129
200
|
```python
|
|
201
|
+
# Basic queries
|
|
130
202
|
js("document.title") # Get page title
|
|
131
|
-
js("document.body.innerText.length") # Get text length
|
|
132
203
|
js("[...document.links].map(a => a.href)") # Get all links
|
|
133
|
-
js("
|
|
204
|
+
js("document.body.innerText.length") # Text length
|
|
205
|
+
|
|
206
|
+
# Async operations
|
|
207
|
+
js("fetch('/api').then(r => r.json())", await_promise=True)
|
|
208
|
+
|
|
209
|
+
# DOM manipulation (no return)
|
|
134
210
|
js("document.querySelectorAll('.ad').forEach(e => e.remove())", wait_return=False)
|
|
135
|
-
|
|
211
|
+
|
|
212
|
+
# Persistent state for multi-step operations
|
|
213
|
+
js("var apiData = null", persist=True)
|
|
214
|
+
js("fetch('/api').then(r => r.json()).then(d => apiData = d)", persist=True, await_promise=True)
|
|
215
|
+
js("apiData.users.length", persist=True)
|
|
136
216
|
```
|
|
137
217
|
|
|
138
218
|
#### Tips
|
|
139
|
-
- **
|
|
140
|
-
- **
|
|
141
|
-
- **
|
|
142
|
-
- **
|
|
143
|
-
- **Hook fetch:** `js("window.fetch = new Proxy(fetch, {apply: (t, _, a) => {console.log(a); return t(...a)}})", wait_return=False)`
|
|
219
|
+
- **Fresh scope:** Default behavior prevents const/let redeclaration errors
|
|
220
|
+
- **Persistent state:** Use `persist=True` for multi-step operations or global hooks
|
|
221
|
+
- **No return needed:** Set `wait_return=False` for DOM manipulation or hooks
|
|
222
|
+
- **Browser selections:** Use `selection=N` with browser() to operate on selected elements
|
|
144
223
|
- **Check console:** `console()` - see logged messages from JS execution
|
|
224
|
+
- **Hook fetch:** `js("window.fetch = new Proxy(fetch, {apply: (t, _, a) => {console.log(a); return t(...a)}})", persist=True, wait_return=False)`
|
|
145
225
|
|
|
146
226
|
### fetch
|
|
147
227
|
Control request interception for debugging and modification.
|
|
@@ -172,6 +252,29 @@ Show paused requests and responses.
|
|
|
172
252
|
- **Modify request:** `resume({id}, modifications={'url': '...'})`
|
|
173
253
|
- **Fail request:** `fail({id}, 'BlockedByClient')` - block the request
|
|
174
254
|
|
|
255
|
+
### page
|
|
256
|
+
Get current page information and navigate.
|
|
257
|
+
|
|
258
|
+
#### Tips
|
|
259
|
+
- **Navigate:** `navigate("https://example.com")` - go to URL
|
|
260
|
+
- **Reload:** `reload()` or `reload(ignore_cache=True)` - refresh page
|
|
261
|
+
- **History:** `back()`, `forward()`, `history()` - navigate history
|
|
262
|
+
- **Execute JS:** `js("document.title")` - run JavaScript in page
|
|
263
|
+
- **Monitor traffic:** `network()` - see requests, `console()` - see messages
|
|
264
|
+
- **Switch page:** `pages()` then `connect(page=N)` - change to another tab
|
|
265
|
+
- **Full status:** `status()` - connection details and event count
|
|
266
|
+
|
|
267
|
+
### pages
|
|
268
|
+
List available Chrome pages and manage connections.
|
|
269
|
+
|
|
270
|
+
#### Tips
|
|
271
|
+
- **Connect to page:** `connect(page={index})` - connect by index number
|
|
272
|
+
- **Connect by ID:** `connect(page_id="{page_id}")` - stable across tab reordering
|
|
273
|
+
- **Switch pages:** Just call `connect()` again - no need to disconnect first
|
|
274
|
+
- **Check status:** `status()` - see current connection and event count
|
|
275
|
+
- **Reconnect:** If connection lost, select page and `connect()` again
|
|
276
|
+
- **Find page:** Look for title/URL in table - index stays consistent
|
|
277
|
+
|
|
175
278
|
### selections
|
|
176
279
|
Browser element selections with prompt and analysis.
|
|
177
280
|
|
webtap/commands/_builders.py
CHANGED
|
@@ -22,7 +22,7 @@ Examples:
|
|
|
22
22
|
|
|
23
23
|
Available builders:
|
|
24
24
|
- table_response() - Tables with headers, warnings, tips
|
|
25
|
-
- info_response() - Key-value pairs with optional heading
|
|
25
|
+
- info_response() - Key-value pairs with optional heading and tips
|
|
26
26
|
- error_response() - Errors with suggestions
|
|
27
27
|
- success_response() - Success messages with details
|
|
28
28
|
- warning_response() - Warnings with suggestions
|
|
@@ -83,6 +83,7 @@ def info_response(
|
|
|
83
83
|
title: str | None = None,
|
|
84
84
|
fields: dict | None = None,
|
|
85
85
|
extra: str | None = None,
|
|
86
|
+
tips: list[str] | None = None,
|
|
86
87
|
) -> dict:
|
|
87
88
|
"""Build info display with key-value pairs.
|
|
88
89
|
|
|
@@ -90,6 +91,7 @@ def info_response(
|
|
|
90
91
|
title: Optional info title
|
|
91
92
|
fields: Dict of field names to values
|
|
92
93
|
extra: Optional extra content (raw markdown)
|
|
94
|
+
tips: Optional developer tips/guidance
|
|
93
95
|
"""
|
|
94
96
|
elements = []
|
|
95
97
|
|
|
@@ -107,6 +109,10 @@ def info_response(
|
|
|
107
109
|
if not elements:
|
|
108
110
|
elements.append({"type": "text", "content": "_No information available_"})
|
|
109
111
|
|
|
112
|
+
if tips:
|
|
113
|
+
elements.append({"type": "heading", "content": "Next Steps", "level": 3})
|
|
114
|
+
elements.append({"type": "list", "items": tips})
|
|
115
|
+
|
|
110
116
|
return {"elements": elements}
|
|
111
117
|
|
|
112
118
|
|