webtap-tool 0.1.1__tar.gz → 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of webtap-tool might be problematic. Click here for more details.
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/CHANGELOG.md +34 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/PKG-INFO +146 -46
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/README.md +145 -45
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/extension/popup.html +9 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/extension/popup.js +28 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/pyproject.toml +1 -1
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/__init__.py +6 -2
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/api.py +71 -3
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/app.py +3 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/cdp/query.py +2 -1
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/commands/launch.py +3 -17
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/services/body.py +1 -1
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/services/console.py +1 -1
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/services/fetch.py +1 -1
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/services/network.py +2 -2
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/services/setup.py +20 -10
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/.gitignore +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/ARCHITECTURE.md +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/data/filters.json +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/extension/manifest.json +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/llms.txt +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/VISION.md +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/cdp/README.md +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/cdp/__init__.py +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/cdp/schema/README.md +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/cdp/schema/cdp_protocol.json +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/cdp/schema/cdp_version.json +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/cdp/session.py +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/commands/DEVELOPER_GUIDE.md +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/commands/TIPS.md +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/commands/__init__.py +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/commands/_builders.py +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/commands/_errors.py +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/commands/_tips.py +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/commands/_utils.py +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/commands/body.py +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/commands/connection.py +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/commands/console.py +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/commands/events.py +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/commands/fetch.py +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/commands/filters.py +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/commands/inspect.py +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/commands/javascript.py +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/commands/navigation.py +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/commands/network.py +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/commands/setup.py +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/filters.py +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/services/README.md +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/services/__init__.py +0 -0
- {webtap_tool-0.1.1 → webtap_tool-0.1.3}/src/webtap/services/main.py +0 -0
|
@@ -15,6 +15,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
15
15
|
|
|
16
16
|
### Removed
|
|
17
17
|
|
|
18
|
+
## [0.1.3] - 2025-09-05
|
|
19
|
+
|
|
20
|
+
### Added
|
|
21
|
+
- Multi-instance WebTap support with first-come-first-serve port management
|
|
22
|
+
- `/instance` endpoint to show WebTap instance information (PID, connected page, events)
|
|
23
|
+
- `/release` endpoint for graceful API port handoff between instances
|
|
24
|
+
- Chrome extension "Switch WebTap Instance" button for managing multiple instances
|
|
25
|
+
- Instance status display in extension popup showing PID and event count
|
|
26
|
+
- Automatic reconnection in extension after instance switching
|
|
27
|
+
|
|
28
|
+
### Changed
|
|
29
|
+
- API server now checks port availability before starting (port 8765)
|
|
30
|
+
- Chrome profile launch strategy uses bindfs mounting instead of symlinks
|
|
31
|
+
- Service docstrings simplified (removed redundant "Internal service for" prefixes)
|
|
32
|
+
- Added `api_thread` tracking to WebTapState for proper thread lifecycle management
|
|
33
|
+
|
|
34
|
+
### Fixed
|
|
35
|
+
- SQL numeric comparisons in query builder now use string comparison instead of CAST
|
|
36
|
+
- Type annotations improved with proper union types (`threading.Thread | None`)
|
|
37
|
+
- Network service HTTP status filtering uses string comparison to prevent SQL errors
|
|
38
|
+
- Extension connection handling during instance transitions
|
|
39
|
+
|
|
40
|
+
### Removed
|
|
41
|
+
|
|
42
|
+
## [0.1.2] - 2025-09-05
|
|
43
|
+
|
|
44
|
+
### Added
|
|
45
|
+
|
|
46
|
+
### Changed
|
|
47
|
+
|
|
48
|
+
### Fixed
|
|
49
|
+
|
|
50
|
+
### Removed
|
|
51
|
+
|
|
18
52
|
## [0.1.1] - 2025-09-05
|
|
19
53
|
|
|
20
54
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: webtap-tool
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: Terminal-based web page inspector for AI debugging sessions
|
|
5
5
|
Author-email: Fredrik Angelsen <fredrikangelsen@gmail.com>
|
|
6
6
|
Classifier: Development Status :: 3 - Alpha
|
|
@@ -25,69 +25,148 @@ Requires-Dist: websocket-client>=1.8.0
|
|
|
25
25
|
Requires-Dist: websockets>=15.0.1
|
|
26
26
|
Description-Content-Type: text/markdown
|
|
27
27
|
|
|
28
|
-
#
|
|
28
|
+
# webtap
|
|
29
29
|
|
|
30
30
|
Browser debugging via Chrome DevTools Protocol with native event storage and dynamic querying.
|
|
31
31
|
|
|
32
|
-
##
|
|
32
|
+
## ✨ Features
|
|
33
33
|
|
|
34
|
-
|
|
34
|
+
- 🔍 **Native CDP Storage** - Events stored exactly as received in DuckDB
|
|
35
|
+
- 🎯 **Dynamic Field Discovery** - Automatically indexes all field paths from events
|
|
36
|
+
- 🚫 **Smart Filtering** - Built-in filters for ads, tracking, analytics noise
|
|
37
|
+
- 📊 **SQL Querying** - Direct DuckDB access for complex analysis
|
|
38
|
+
- 🔌 **MCP Ready** - Tools and resources for Claude/LLMs
|
|
39
|
+
- 🎨 **Rich Display** - Tables, alerts, and formatted output
|
|
40
|
+
- 🐍 **Python Inspection** - Full Python environment for data exploration
|
|
35
41
|
|
|
36
|
-
##
|
|
42
|
+
## 📋 Prerequisites
|
|
37
43
|
|
|
38
|
-
|
|
39
|
-
- **
|
|
40
|
-
- **Smart Filtering** - Built-in filters for ads, tracking, analytics noise
|
|
41
|
-
- **SQL Querying** - Direct DuckDB access for complex analysis
|
|
42
|
-
- **Chrome Extension** - Visual page selector and connection management
|
|
43
|
-
- **Python Inspection** - Full Python environment for data exploration
|
|
44
|
-
|
|
45
|
-
## Installation
|
|
44
|
+
Required system dependencies:
|
|
45
|
+
- **google-chrome-stable** or **chromium** - Browser with DevTools Protocol support
|
|
46
46
|
|
|
47
47
|
```bash
|
|
48
|
-
#
|
|
49
|
-
|
|
48
|
+
# macOS
|
|
49
|
+
brew install --cask google-chrome
|
|
50
|
+
|
|
51
|
+
# Ubuntu/Debian
|
|
52
|
+
wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
|
|
53
|
+
sudo sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list'
|
|
54
|
+
sudo apt update
|
|
55
|
+
sudo apt install google-chrome-stable
|
|
50
56
|
|
|
51
|
-
#
|
|
52
|
-
|
|
53
|
-
|
|
57
|
+
# Arch Linux
|
|
58
|
+
yay -S google-chrome # or google-chrome-stable from AUR
|
|
59
|
+
|
|
60
|
+
# Fedora
|
|
61
|
+
sudo dnf install google-chrome-stable
|
|
54
62
|
```
|
|
55
63
|
|
|
56
|
-
##
|
|
64
|
+
## 📦 Installation
|
|
57
65
|
|
|
58
|
-
1. **Start Chrome with debugging**
|
|
59
66
|
```bash
|
|
60
|
-
#
|
|
61
|
-
|
|
67
|
+
# Install via uv tool (recommended)
|
|
68
|
+
uv tool install webtap-tool
|
|
62
69
|
|
|
63
|
-
#
|
|
64
|
-
|
|
70
|
+
# Or with pipx
|
|
71
|
+
pipx install webtap-tool
|
|
65
72
|
|
|
66
|
-
#
|
|
67
|
-
|
|
73
|
+
# Update to latest
|
|
74
|
+
uv tool upgrade webtap-tool
|
|
75
|
+
|
|
76
|
+
# Uninstall
|
|
77
|
+
uv tool uninstall webtap-tool
|
|
68
78
|
```
|
|
69
79
|
|
|
70
|
-
|
|
80
|
+
## 🚀 Quick Start
|
|
81
|
+
|
|
71
82
|
```bash
|
|
83
|
+
# 1. Install webtap
|
|
84
|
+
uv tool install webtap-tool
|
|
85
|
+
|
|
86
|
+
# 2. Optional: Setup helpers (first time only)
|
|
87
|
+
webtap --cli setup-filters # Download default filter configurations
|
|
88
|
+
webtap --cli setup-extension # Download Chrome extension files
|
|
89
|
+
webtap --cli setup-chrome # Install Chrome wrapper for debugging
|
|
90
|
+
|
|
91
|
+
# 3. Launch Chrome with debugging
|
|
92
|
+
webtap --cli run-chrome # Or manually: google-chrome-stable --remote-debugging-port=9222
|
|
93
|
+
|
|
94
|
+
# 4. Start webtap REPL
|
|
72
95
|
webtap
|
|
73
96
|
|
|
74
|
-
#
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
>>>
|
|
97
|
+
# 5. Connect and explore
|
|
98
|
+
>>> pages() # List available Chrome pages
|
|
99
|
+
>>> connect(0) # Connect to first page
|
|
100
|
+
>>> network() # View network requests (filtered)
|
|
101
|
+
>>> console() # View console messages
|
|
102
|
+
>>> events({"url": "*api*"}) # Query any CDP field dynamically
|
|
80
103
|
```
|
|
81
104
|
|
|
82
|
-
|
|
105
|
+
## 🔌 MCP Setup for Claude
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
# Quick setup with Claude CLI
|
|
109
|
+
claude mcp add webtap -- webtap --mcp
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
Or manually configure Claude Desktop (`~/Library/Application Support/Claude/claude_desktop_config.json`):
|
|
113
|
+
```json
|
|
114
|
+
{
|
|
115
|
+
"mcpServers": {
|
|
116
|
+
"webtap": {
|
|
117
|
+
"command": "webtap",
|
|
118
|
+
"args": ["--mcp"]
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
## 🎮 Usage
|
|
125
|
+
|
|
126
|
+
### Interactive REPL
|
|
127
|
+
```bash
|
|
128
|
+
webtap # Start REPL
|
|
129
|
+
webtap --mcp # Start as MCP server
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
### CLI Commands
|
|
133
|
+
```bash
|
|
134
|
+
webtap --cli setup-filters # Download filter configurations
|
|
135
|
+
webtap --cli setup-extension # Download Chrome extension
|
|
136
|
+
webtap --cli setup-chrome # Install Chrome wrapper script
|
|
137
|
+
webtap --cli run-chrome # Launch Chrome with debugging
|
|
138
|
+
webtap --cli --help # Show all CLI commands
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
### Commands
|
|
83
142
|
```python
|
|
84
143
|
>>> pages() # List available Chrome pages
|
|
85
144
|
>>> connect(0) # Connect to first page
|
|
86
145
|
>>> network() # View network requests (filtered)
|
|
87
146
|
>>> console() # View console messages
|
|
88
147
|
>>> events({"url": "*api*"}) # Query any CDP field dynamically
|
|
148
|
+
>>> body(50) # Get response body
|
|
149
|
+
>>> inspect(49) # View event details
|
|
150
|
+
>>> js("document.title") # Execute JavaScript
|
|
89
151
|
```
|
|
90
152
|
|
|
153
|
+
### Command Reference
|
|
154
|
+
|
|
155
|
+
| Command | Description |
|
|
156
|
+
|---------|------------|
|
|
157
|
+
| `pages()` | List available Chrome pages |
|
|
158
|
+
| `connect(page=0)` | Connect to page by index |
|
|
159
|
+
| `disconnect()` | Disconnect from current page |
|
|
160
|
+
| `navigate(url)` | Navigate to URL |
|
|
161
|
+
| `network(no_filters=False)` | View network requests |
|
|
162
|
+
| `console()` | View console messages |
|
|
163
|
+
| `events(filters)` | Query events dynamically |
|
|
164
|
+
| `inspect(rowid, expr=None)` | Inspect event details |
|
|
165
|
+
| `body(response_id, expr=None)` | Get response body |
|
|
166
|
+
| `js(code, wait_return=True)` | Execute JavaScript |
|
|
167
|
+
| `filters(action="list")` | Manage noise filters |
|
|
168
|
+
| `clear(events=True)` | Clear events/console/cache |
|
|
169
|
+
|
|
91
170
|
## Core Commands
|
|
92
171
|
|
|
93
172
|
### Connection & Navigation
|
|
@@ -392,20 +471,41 @@ WebTap includes aggressive default filters to reduce noise. Customize in `.webta
|
|
|
392
471
|
- Python 3.12+
|
|
393
472
|
- Dependencies: websocket-client, duckdb, replkit2, fastapi, uvicorn, beautifulsoup4
|
|
394
473
|
|
|
395
|
-
##
|
|
474
|
+
## 🏗️ Architecture
|
|
475
|
+
|
|
476
|
+
Built on [ReplKit2](https://github.com/angelsen/replkit2) for dual REPL/MCP functionality.
|
|
477
|
+
|
|
478
|
+
**Key Design:**
|
|
479
|
+
- **Store AS-IS** - No transformation of CDP events
|
|
480
|
+
- **Query On-Demand** - Extract only what's needed
|
|
481
|
+
- **Dynamic Discovery** - No predefined schemas
|
|
482
|
+
- **SQL-First** - Leverage DuckDB's JSON capabilities
|
|
483
|
+
- **Minimal Memory** - Store only CDP data
|
|
484
|
+
|
|
485
|
+
## 📚 Documentation
|
|
486
|
+
|
|
487
|
+
- [Architecture](ARCHITECTURE.md) - System design
|
|
488
|
+
- [Vision](src/webtap/VISION.md) - Design philosophy
|
|
489
|
+
- [Services](src/webtap/services/) - Service layer implementations
|
|
490
|
+
- [Commands](src/webtap/commands/) - Command implementations
|
|
491
|
+
|
|
492
|
+
## 🛠️ Development
|
|
396
493
|
|
|
397
494
|
```bash
|
|
398
|
-
#
|
|
399
|
-
|
|
400
|
-
|
|
495
|
+
# Clone repository
|
|
496
|
+
git clone https://github.com/angelsen/tap-tools
|
|
497
|
+
cd tap-tools
|
|
498
|
+
|
|
499
|
+
# Install for development
|
|
500
|
+
uv sync --package webtap
|
|
401
501
|
|
|
402
|
-
#
|
|
403
|
-
|
|
502
|
+
# Run development version
|
|
503
|
+
uv run --package webtap webtap
|
|
404
504
|
|
|
405
|
-
#
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
505
|
+
# Run tests and checks
|
|
506
|
+
make check-webtap # Check build
|
|
507
|
+
make format # Format code
|
|
508
|
+
make lint # Fix linting
|
|
409
509
|
```
|
|
410
510
|
|
|
411
511
|
## API Server
|
|
@@ -422,6 +522,6 @@ WebTap automatically starts a FastAPI server on port 8765 for Chrome extension i
|
|
|
422
522
|
|
|
423
523
|
The API server runs in a background thread and doesn't block the REPL.
|
|
424
524
|
|
|
425
|
-
## License
|
|
525
|
+
## 📄 License
|
|
426
526
|
|
|
427
|
-
MIT -
|
|
527
|
+
MIT - see [LICENSE](../../LICENSE) for details.
|
|
@@ -1,66 +1,145 @@
|
|
|
1
|
-
#
|
|
1
|
+
# webtap
|
|
2
2
|
|
|
3
3
|
Browser debugging via Chrome DevTools Protocol with native event storage and dynamic querying.
|
|
4
4
|
|
|
5
|
-
##
|
|
5
|
+
## ✨ Features
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
- 🔍 **Native CDP Storage** - Events stored exactly as received in DuckDB
|
|
8
|
+
- 🎯 **Dynamic Field Discovery** - Automatically indexes all field paths from events
|
|
9
|
+
- 🚫 **Smart Filtering** - Built-in filters for ads, tracking, analytics noise
|
|
10
|
+
- 📊 **SQL Querying** - Direct DuckDB access for complex analysis
|
|
11
|
+
- 🔌 **MCP Ready** - Tools and resources for Claude/LLMs
|
|
12
|
+
- 🎨 **Rich Display** - Tables, alerts, and formatted output
|
|
13
|
+
- 🐍 **Python Inspection** - Full Python environment for data exploration
|
|
8
14
|
|
|
9
|
-
##
|
|
15
|
+
## 📋 Prerequisites
|
|
10
16
|
|
|
11
|
-
|
|
12
|
-
- **
|
|
13
|
-
- **Smart Filtering** - Built-in filters for ads, tracking, analytics noise
|
|
14
|
-
- **SQL Querying** - Direct DuckDB access for complex analysis
|
|
15
|
-
- **Chrome Extension** - Visual page selector and connection management
|
|
16
|
-
- **Python Inspection** - Full Python environment for data exploration
|
|
17
|
-
|
|
18
|
-
## Installation
|
|
17
|
+
Required system dependencies:
|
|
18
|
+
- **google-chrome-stable** or **chromium** - Browser with DevTools Protocol support
|
|
19
19
|
|
|
20
20
|
```bash
|
|
21
|
-
#
|
|
22
|
-
|
|
21
|
+
# macOS
|
|
22
|
+
brew install --cask google-chrome
|
|
23
|
+
|
|
24
|
+
# Ubuntu/Debian
|
|
25
|
+
wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
|
|
26
|
+
sudo sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list'
|
|
27
|
+
sudo apt update
|
|
28
|
+
sudo apt install google-chrome-stable
|
|
23
29
|
|
|
24
|
-
#
|
|
25
|
-
|
|
26
|
-
|
|
30
|
+
# Arch Linux
|
|
31
|
+
yay -S google-chrome # or google-chrome-stable from AUR
|
|
32
|
+
|
|
33
|
+
# Fedora
|
|
34
|
+
sudo dnf install google-chrome-stable
|
|
27
35
|
```
|
|
28
36
|
|
|
29
|
-
##
|
|
37
|
+
## 📦 Installation
|
|
30
38
|
|
|
31
|
-
1. **Start Chrome with debugging**
|
|
32
39
|
```bash
|
|
33
|
-
#
|
|
34
|
-
|
|
40
|
+
# Install via uv tool (recommended)
|
|
41
|
+
uv tool install webtap-tool
|
|
35
42
|
|
|
36
|
-
#
|
|
37
|
-
|
|
43
|
+
# Or with pipx
|
|
44
|
+
pipx install webtap-tool
|
|
38
45
|
|
|
39
|
-
#
|
|
40
|
-
|
|
46
|
+
# Update to latest
|
|
47
|
+
uv tool upgrade webtap-tool
|
|
48
|
+
|
|
49
|
+
# Uninstall
|
|
50
|
+
uv tool uninstall webtap-tool
|
|
41
51
|
```
|
|
42
52
|
|
|
43
|
-
|
|
53
|
+
## 🚀 Quick Start
|
|
54
|
+
|
|
44
55
|
```bash
|
|
56
|
+
# 1. Install webtap
|
|
57
|
+
uv tool install webtap-tool
|
|
58
|
+
|
|
59
|
+
# 2. Optional: Setup helpers (first time only)
|
|
60
|
+
webtap --cli setup-filters # Download default filter configurations
|
|
61
|
+
webtap --cli setup-extension # Download Chrome extension files
|
|
62
|
+
webtap --cli setup-chrome # Install Chrome wrapper for debugging
|
|
63
|
+
|
|
64
|
+
# 3. Launch Chrome with debugging
|
|
65
|
+
webtap --cli run-chrome # Or manually: google-chrome-stable --remote-debugging-port=9222
|
|
66
|
+
|
|
67
|
+
# 4. Start webtap REPL
|
|
45
68
|
webtap
|
|
46
69
|
|
|
47
|
-
#
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
>>>
|
|
70
|
+
# 5. Connect and explore
|
|
71
|
+
>>> pages() # List available Chrome pages
|
|
72
|
+
>>> connect(0) # Connect to first page
|
|
73
|
+
>>> network() # View network requests (filtered)
|
|
74
|
+
>>> console() # View console messages
|
|
75
|
+
>>> events({"url": "*api*"}) # Query any CDP field dynamically
|
|
53
76
|
```
|
|
54
77
|
|
|
55
|
-
|
|
78
|
+
## 🔌 MCP Setup for Claude
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
# Quick setup with Claude CLI
|
|
82
|
+
claude mcp add webtap -- webtap --mcp
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Or manually configure Claude Desktop (`~/Library/Application Support/Claude/claude_desktop_config.json`):
|
|
86
|
+
```json
|
|
87
|
+
{
|
|
88
|
+
"mcpServers": {
|
|
89
|
+
"webtap": {
|
|
90
|
+
"command": "webtap",
|
|
91
|
+
"args": ["--mcp"]
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
## 🎮 Usage
|
|
98
|
+
|
|
99
|
+
### Interactive REPL
|
|
100
|
+
```bash
|
|
101
|
+
webtap # Start REPL
|
|
102
|
+
webtap --mcp # Start as MCP server
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
### CLI Commands
|
|
106
|
+
```bash
|
|
107
|
+
webtap --cli setup-filters # Download filter configurations
|
|
108
|
+
webtap --cli setup-extension # Download Chrome extension
|
|
109
|
+
webtap --cli setup-chrome # Install Chrome wrapper script
|
|
110
|
+
webtap --cli run-chrome # Launch Chrome with debugging
|
|
111
|
+
webtap --cli --help # Show all CLI commands
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
### Commands
|
|
56
115
|
```python
|
|
57
116
|
>>> pages() # List available Chrome pages
|
|
58
117
|
>>> connect(0) # Connect to first page
|
|
59
118
|
>>> network() # View network requests (filtered)
|
|
60
119
|
>>> console() # View console messages
|
|
61
120
|
>>> events({"url": "*api*"}) # Query any CDP field dynamically
|
|
121
|
+
>>> body(50) # Get response body
|
|
122
|
+
>>> inspect(49) # View event details
|
|
123
|
+
>>> js("document.title") # Execute JavaScript
|
|
62
124
|
```
|
|
63
125
|
|
|
126
|
+
### Command Reference
|
|
127
|
+
|
|
128
|
+
| Command | Description |
|
|
129
|
+
|---------|------------|
|
|
130
|
+
| `pages()` | List available Chrome pages |
|
|
131
|
+
| `connect(page=0)` | Connect to page by index |
|
|
132
|
+
| `disconnect()` | Disconnect from current page |
|
|
133
|
+
| `navigate(url)` | Navigate to URL |
|
|
134
|
+
| `network(no_filters=False)` | View network requests |
|
|
135
|
+
| `console()` | View console messages |
|
|
136
|
+
| `events(filters)` | Query events dynamically |
|
|
137
|
+
| `inspect(rowid, expr=None)` | Inspect event details |
|
|
138
|
+
| `body(response_id, expr=None)` | Get response body |
|
|
139
|
+
| `js(code, wait_return=True)` | Execute JavaScript |
|
|
140
|
+
| `filters(action="list")` | Manage noise filters |
|
|
141
|
+
| `clear(events=True)` | Clear events/console/cache |
|
|
142
|
+
|
|
64
143
|
## Core Commands
|
|
65
144
|
|
|
66
145
|
### Connection & Navigation
|
|
@@ -365,20 +444,41 @@ WebTap includes aggressive default filters to reduce noise. Customize in `.webta
|
|
|
365
444
|
- Python 3.12+
|
|
366
445
|
- Dependencies: websocket-client, duckdb, replkit2, fastapi, uvicorn, beautifulsoup4
|
|
367
446
|
|
|
368
|
-
##
|
|
447
|
+
## 🏗️ Architecture
|
|
448
|
+
|
|
449
|
+
Built on [ReplKit2](https://github.com/angelsen/replkit2) for dual REPL/MCP functionality.
|
|
450
|
+
|
|
451
|
+
**Key Design:**
|
|
452
|
+
- **Store AS-IS** - No transformation of CDP events
|
|
453
|
+
- **Query On-Demand** - Extract only what's needed
|
|
454
|
+
- **Dynamic Discovery** - No predefined schemas
|
|
455
|
+
- **SQL-First** - Leverage DuckDB's JSON capabilities
|
|
456
|
+
- **Minimal Memory** - Store only CDP data
|
|
457
|
+
|
|
458
|
+
## 📚 Documentation
|
|
459
|
+
|
|
460
|
+
- [Architecture](ARCHITECTURE.md) - System design
|
|
461
|
+
- [Vision](src/webtap/VISION.md) - Design philosophy
|
|
462
|
+
- [Services](src/webtap/services/) - Service layer implementations
|
|
463
|
+
- [Commands](src/webtap/commands/) - Command implementations
|
|
464
|
+
|
|
465
|
+
## 🛠️ Development
|
|
369
466
|
|
|
370
467
|
```bash
|
|
371
|
-
#
|
|
372
|
-
|
|
373
|
-
|
|
468
|
+
# Clone repository
|
|
469
|
+
git clone https://github.com/angelsen/tap-tools
|
|
470
|
+
cd tap-tools
|
|
471
|
+
|
|
472
|
+
# Install for development
|
|
473
|
+
uv sync --package webtap
|
|
374
474
|
|
|
375
|
-
#
|
|
376
|
-
|
|
475
|
+
# Run development version
|
|
476
|
+
uv run --package webtap webtap
|
|
377
477
|
|
|
378
|
-
#
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
478
|
+
# Run tests and checks
|
|
479
|
+
make check-webtap # Check build
|
|
480
|
+
make format # Format code
|
|
481
|
+
make lint # Fix linting
|
|
382
482
|
```
|
|
383
483
|
|
|
384
484
|
## API Server
|
|
@@ -395,6 +495,6 @@ WebTap automatically starts a FastAPI server on port 8765 for Chrome extension i
|
|
|
395
495
|
|
|
396
496
|
The API server runs in a background thread and doesn't block the REPL.
|
|
397
497
|
|
|
398
|
-
## License
|
|
498
|
+
## 📄 License
|
|
399
499
|
|
|
400
|
-
MIT -
|
|
500
|
+
MIT - see [LICENSE](../../LICENSE) for details.
|
|
@@ -150,6 +150,15 @@
|
|
|
150
150
|
|
|
151
151
|
<div class="divider"></div>
|
|
152
152
|
|
|
153
|
+
<button
|
|
154
|
+
id="switchInstance"
|
|
155
|
+
style="width: 100%; font-size: 11px; margin-bottom: 8px"
|
|
156
|
+
>
|
|
157
|
+
Switch WebTap Instance
|
|
158
|
+
</button>
|
|
159
|
+
|
|
160
|
+
<div class="divider"></div>
|
|
161
|
+
|
|
153
162
|
<div class="filter-section">
|
|
154
163
|
<div
|
|
155
164
|
style="
|
|
@@ -25,6 +25,13 @@ async function loadPages() {
|
|
|
25
25
|
return;
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
+
// Also fetch instance info if available
|
|
29
|
+
const instanceInfo = await api("/instance");
|
|
30
|
+
if (instanceInfo && !instanceInfo.error) {
|
|
31
|
+
document.getElementById("switchInstance").title =
|
|
32
|
+
`PID: ${instanceInfo.pid} | Events: ${instanceInfo.events}`;
|
|
33
|
+
}
|
|
34
|
+
|
|
28
35
|
const pages = result.pages || [];
|
|
29
36
|
const select = document.getElementById("pageList");
|
|
30
37
|
|
|
@@ -247,6 +254,27 @@ document.getElementById("disableAllFilters").onclick = async () => {
|
|
|
247
254
|
setTimeout(updateFilters, 100);
|
|
248
255
|
};
|
|
249
256
|
|
|
257
|
+
// Switch to a different WebTap instance
|
|
258
|
+
document.getElementById("switchInstance").onclick = async () => {
|
|
259
|
+
const result = await api("/release", "POST");
|
|
260
|
+
if (!result.error) {
|
|
261
|
+
document.getElementById("status").innerHTML =
|
|
262
|
+
'<span style="color: #666">Port released. Start new WebTap.</span>';
|
|
263
|
+
// Disable controls until reconnected
|
|
264
|
+
document.getElementById("connect").disabled = true;
|
|
265
|
+
document.getElementById("disconnect").disabled = true;
|
|
266
|
+
document.getElementById("fetchToggle").disabled = true;
|
|
267
|
+
// Try to reconnect after delay
|
|
268
|
+
setTimeout(() => {
|
|
269
|
+
loadPages();
|
|
270
|
+
updateStatus();
|
|
271
|
+
}, 2000);
|
|
272
|
+
} else {
|
|
273
|
+
document.getElementById("status").innerHTML =
|
|
274
|
+
`<span class="error">Error: ${result.error}</span>`;
|
|
275
|
+
}
|
|
276
|
+
};
|
|
277
|
+
|
|
250
278
|
// Update all status from server - single source of truth
|
|
251
279
|
async function updateStatus() {
|
|
252
280
|
const status = await api("/status");
|
|
@@ -47,8 +47,12 @@ def main():
|
|
|
47
47
|
def _start_api_server_safely():
|
|
48
48
|
"""Start API server with error handling."""
|
|
49
49
|
try:
|
|
50
|
-
start_api_server(app.state)
|
|
51
|
-
|
|
50
|
+
thread = start_api_server(app.state)
|
|
51
|
+
if thread and app.state:
|
|
52
|
+
app.state.api_thread = thread
|
|
53
|
+
logger.info("API server started on port 8765")
|
|
54
|
+
else:
|
|
55
|
+
logger.info("Port 8765 in use by another instance")
|
|
52
56
|
except Exception as e:
|
|
53
57
|
logger.warning(f"Failed to start API server: {e}")
|
|
54
58
|
|
|
@@ -5,6 +5,8 @@ PUBLIC API:
|
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
7
|
import logging
|
|
8
|
+
import os
|
|
9
|
+
import socket
|
|
8
10
|
import threading
|
|
9
11
|
from typing import Any, Dict
|
|
10
12
|
|
|
@@ -184,7 +186,39 @@ async def disable_all_filters() -> Dict[str, Any]:
|
|
|
184
186
|
return {"enabled": [], "total": 0}
|
|
185
187
|
|
|
186
188
|
|
|
187
|
-
|
|
189
|
+
@api.get("/instance")
|
|
190
|
+
async def get_instance_info() -> Dict[str, Any]:
|
|
191
|
+
"""Get info about this WebTap instance."""
|
|
192
|
+
if not app_state:
|
|
193
|
+
return {"error": "WebTap not initialized"}
|
|
194
|
+
|
|
195
|
+
return {
|
|
196
|
+
"pid": os.getpid(),
|
|
197
|
+
"connected_to": app_state.cdp.current_page_title if app_state.cdp.is_connected else None,
|
|
198
|
+
"events": app_state.cdp.event_count,
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
@api.post("/release")
|
|
203
|
+
async def release_port() -> Dict[str, Any]:
|
|
204
|
+
"""Release API port for another WebTap instance."""
|
|
205
|
+
logger.info("Releasing API port for another instance")
|
|
206
|
+
|
|
207
|
+
# Schedule graceful shutdown after response
|
|
208
|
+
def shutdown():
|
|
209
|
+
# Just set the flag to stop uvicorn, don't kill the whole process
|
|
210
|
+
global _shutdown_requested
|
|
211
|
+
_shutdown_requested = True
|
|
212
|
+
|
|
213
|
+
threading.Timer(0.5, shutdown).start()
|
|
214
|
+
return {"message": "Releasing port 8765"}
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
# Flag to signal shutdown
|
|
218
|
+
_shutdown_requested = False
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def start_api_server(state, host: str = "127.0.0.1", port: int = 8765) -> threading.Thread | None:
|
|
188
222
|
"""Start the API server in a background thread.
|
|
189
223
|
|
|
190
224
|
Args:
|
|
@@ -193,8 +227,16 @@ def start_api_server(state, host: str = "127.0.0.1", port: int = 8765):
|
|
|
193
227
|
port: Port to bind to. Defaults to 8765.
|
|
194
228
|
|
|
195
229
|
Returns:
|
|
196
|
-
Thread instance running the server.
|
|
230
|
+
Thread instance running the server, or None if port is in use.
|
|
197
231
|
"""
|
|
232
|
+
# Check port availability first
|
|
233
|
+
try:
|
|
234
|
+
with socket.socket() as s:
|
|
235
|
+
s.bind((host, port))
|
|
236
|
+
except OSError:
|
|
237
|
+
logger.info(f"Port {port} already in use")
|
|
238
|
+
return None
|
|
239
|
+
|
|
198
240
|
global app_state
|
|
199
241
|
app_state = state
|
|
200
242
|
|
|
@@ -208,13 +250,39 @@ def start_api_server(state, host: str = "127.0.0.1", port: int = 8765):
|
|
|
208
250
|
def run_server(host: str, port: int):
|
|
209
251
|
"""Run the FastAPI server in a thread."""
|
|
210
252
|
try:
|
|
211
|
-
uvicorn.
|
|
253
|
+
config = uvicorn.Config(
|
|
212
254
|
api,
|
|
213
255
|
host=host,
|
|
214
256
|
port=port,
|
|
215
257
|
log_level="error",
|
|
216
258
|
access_log=False,
|
|
217
259
|
)
|
|
260
|
+
server = uvicorn.Server(config)
|
|
261
|
+
|
|
262
|
+
# Run with checking for shutdown flag
|
|
263
|
+
import asyncio
|
|
264
|
+
|
|
265
|
+
loop = asyncio.new_event_loop()
|
|
266
|
+
asyncio.set_event_loop(loop)
|
|
267
|
+
|
|
268
|
+
async def serve():
|
|
269
|
+
await server.serve()
|
|
270
|
+
|
|
271
|
+
# Start serving
|
|
272
|
+
task = loop.create_task(serve())
|
|
273
|
+
|
|
274
|
+
# Check for shutdown flag
|
|
275
|
+
while not _shutdown_requested:
|
|
276
|
+
loop.run_until_complete(asyncio.sleep(0.1))
|
|
277
|
+
if task.done():
|
|
278
|
+
break
|
|
279
|
+
|
|
280
|
+
# Shutdown if requested
|
|
281
|
+
if _shutdown_requested:
|
|
282
|
+
logger.info("API server shutting down")
|
|
283
|
+
server.should_exit = True
|
|
284
|
+
loop.run_until_complete(server.shutdown())
|
|
285
|
+
|
|
218
286
|
except Exception as e:
|
|
219
287
|
logger.error(f"API server failed: {e}")
|
|
220
288
|
|
|
@@ -6,6 +6,7 @@ PUBLIC API:
|
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
8
|
import sys
|
|
9
|
+
import threading
|
|
9
10
|
from dataclasses import dataclass, field
|
|
10
11
|
|
|
11
12
|
from replkit2 import App
|
|
@@ -24,10 +25,12 @@ class WebTapState:
|
|
|
24
25
|
Attributes:
|
|
25
26
|
cdp: Chrome DevTools Protocol session instance.
|
|
26
27
|
service: WebTapService orchestrating all domain services.
|
|
28
|
+
api_thread: Thread running the FastAPI server (if this instance owns the port).
|
|
27
29
|
"""
|
|
28
30
|
|
|
29
31
|
cdp: CDPSession = field(default_factory=CDPSession)
|
|
30
32
|
service: WebTapService = field(init=False)
|
|
33
|
+
api_thread: threading.Thread | None = None
|
|
31
34
|
|
|
32
35
|
def __post_init__(self):
|
|
33
36
|
"""Initialize service with self reference after dataclass init."""
|
|
@@ -74,7 +74,8 @@ def build_query(
|
|
|
74
74
|
pattern = value
|
|
75
75
|
path_conditions.append(f"json_extract_string(event, '{json_path}') LIKE '{pattern}'")
|
|
76
76
|
elif isinstance(value, (int, float)):
|
|
77
|
-
|
|
77
|
+
# Use string comparison for numeric values to avoid type conversion errors
|
|
78
|
+
path_conditions.append(f"json_extract_string(event, '{json_path}') = '{value}'")
|
|
78
79
|
elif isinstance(value, bool):
|
|
79
80
|
path_conditions.append(f"json_extract_string(event, '{json_path}') = '{str(value).lower()}'")
|
|
80
81
|
elif value is None:
|
|
@@ -46,23 +46,9 @@ def run_chrome(state, detach: bool = True, port: int = 9222) -> dict:
|
|
|
46
46
|
],
|
|
47
47
|
)
|
|
48
48
|
|
|
49
|
-
#
|
|
49
|
+
# Simple: use clean temp profile for debugging
|
|
50
50
|
temp_config = Path("/tmp/webtap-chrome-debug")
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
if not temp_config.exists():
|
|
54
|
-
temp_config.mkdir(parents=True)
|
|
55
|
-
|
|
56
|
-
# Symlink Default profile
|
|
57
|
-
default_profile = real_config / "Default"
|
|
58
|
-
if default_profile.exists():
|
|
59
|
-
(temp_config / "Default").symlink_to(default_profile)
|
|
60
|
-
|
|
61
|
-
# Copy essential files
|
|
62
|
-
for file in ["Local State", "First Run"]:
|
|
63
|
-
src = real_config / file
|
|
64
|
-
if src.exists():
|
|
65
|
-
(temp_config / file).write_text(src.read_text())
|
|
51
|
+
temp_config.mkdir(parents=True, exist_ok=True)
|
|
66
52
|
|
|
67
53
|
# Launch Chrome
|
|
68
54
|
cmd = [chrome_exe, f"--remote-debugging-port={port}", "--remote-allow-origins=*", f"--user-data-dir={temp_config}"]
|
|
@@ -74,7 +60,7 @@ def run_chrome(state, detach: bool = True, port: int = 9222) -> dict:
|
|
|
74
60
|
details={
|
|
75
61
|
"Port": str(port),
|
|
76
62
|
"Mode": "Background (detached)",
|
|
77
|
-
"Profile":
|
|
63
|
+
"Profile": "Temporary (clean)",
|
|
78
64
|
"Next step": "Run connect() to attach WebTap",
|
|
79
65
|
},
|
|
80
66
|
)
|
|
@@ -10,7 +10,7 @@ logger = logging.getLogger(__name__)
|
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
class NetworkService:
|
|
13
|
-
"""
|
|
13
|
+
"""Network event queries and monitoring."""
|
|
14
14
|
|
|
15
15
|
def __init__(self):
|
|
16
16
|
"""Initialize network service."""
|
|
@@ -81,7 +81,7 @@ class NetworkService:
|
|
|
81
81
|
json_extract_string(event, '$.params.response.statusText') as StatusText
|
|
82
82
|
FROM events
|
|
83
83
|
WHERE json_extract_string(event, '$.method') = 'Network.responseReceived'
|
|
84
|
-
AND
|
|
84
|
+
AND json_extract_string(event, '$.params.response.status') >= '400'
|
|
85
85
|
ORDER BY rowid DESC LIMIT {limit}
|
|
86
86
|
"""
|
|
87
87
|
|
|
@@ -177,22 +177,32 @@ class SetupService:
|
|
|
177
177
|
}
|
|
178
178
|
|
|
179
179
|
wrapper_script = """#!/bin/bash
|
|
180
|
-
# Chrome wrapper
|
|
180
|
+
# Chrome wrapper using bindfs for perfect state sync with debug port
|
|
181
181
|
|
|
182
|
-
|
|
183
|
-
|
|
182
|
+
DEBUG_DIR="$HOME/.config/google-chrome-debug"
|
|
183
|
+
REAL_DIR="$HOME/.config/google-chrome"
|
|
184
184
|
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
cp "$REAL_CONFIG/First Run" "$DEBUG_CONFIG/" 2>/dev/null || true
|
|
185
|
+
# Check if bindfs is installed
|
|
186
|
+
if ! command -v bindfs &>/dev/null; then
|
|
187
|
+
echo "Error: bindfs not installed. Install with: yay -S bindfs" >&2
|
|
188
|
+
exit 1
|
|
190
189
|
fi
|
|
191
190
|
|
|
191
|
+
# Mount real profile via bindfs if not already mounted
|
|
192
|
+
if ! mountpoint -q "$DEBUG_DIR" 2>/dev/null; then
|
|
193
|
+
mkdir -p "$DEBUG_DIR"
|
|
194
|
+
if ! bindfs --no-allow-other "$REAL_DIR" "$DEBUG_DIR"; then
|
|
195
|
+
echo "Error: Failed to mount Chrome profile via bindfs" >&2
|
|
196
|
+
exit 1
|
|
197
|
+
fi
|
|
198
|
+
echo "Chrome debug profile mounted. To unmount: fusermount -u $DEBUG_DIR" >&2
|
|
199
|
+
fi
|
|
200
|
+
|
|
201
|
+
# Launch Chrome with debugging on bindfs mount
|
|
192
202
|
exec /usr/bin/google-chrome-stable \\
|
|
193
203
|
--remote-debugging-port=9222 \\
|
|
194
|
-
--remote-allow-origins
|
|
195
|
-
--user-data-dir="$
|
|
204
|
+
--remote-allow-origins='*' \\
|
|
205
|
+
--user-data-dir="$DEBUG_DIR" \\
|
|
196
206
|
"$@"
|
|
197
207
|
"""
|
|
198
208
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|