webpilot-engine 2.0.5__tar.gz → 2.0.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/PKG-INFO +304 -304
  2. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/README.md +2 -2
  3. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/mcp_server.py +32 -11
  4. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/pyproject.toml +1 -1
  5. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/setup.cfg +4 -4
  6. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/webpilot_engine.egg-info/PKG-INFO +304 -304
  7. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/adapters/__init__.py +0 -0
  8. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/adapters/base.py +0 -0
  9. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/adapters/browser_adapter.py +0 -0
  10. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/adapters/dom_scripts.py +0 -0
  11. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/adapters/envelope_builder.py +0 -0
  12. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/adapters/live_session_manager.py +0 -0
  13. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/adapters/upload_adapter.py +0 -0
  14. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/builder.py +0 -0
  15. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/cli.py +0 -0
  16. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/common/__init__.py +0 -0
  17. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/common/cookies.py +0 -0
  18. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/common/logging.py +0 -0
  19. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/common/process_manager.py +0 -0
  20. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/common/retry.py +0 -0
  21. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/common/secrets.py +0 -0
  22. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/common/text.py +0 -0
  23. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/config/__init__.py +0 -0
  24. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/config/device_profile.example.json +0 -0
  25. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/config/device_profile.json +0 -0
  26. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/config/settings.example.json +0 -0
  27. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/config/settings.json +0 -0
  28. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/config/settings.py +0 -0
  29. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/constants/__init__.py +0 -0
  30. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/constants/contracts.py +0 -0
  31. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/constants/selectors.py +0 -0
  32. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/constants/timeouts.py +0 -0
  33. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/core/__init__.py +0 -0
  34. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/core/exceptions.py +0 -0
  35. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/core/models.py +0 -0
  36. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/engine.py +0 -0
  37. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/exceptions.py +0 -0
  38. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/models.py +0 -0
  39. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/orchestration/__init__.py +0 -0
  40. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/orchestration/context.py +0 -0
  41. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/orchestration/flow_orchestrator.py +0 -0
  42. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/orchestration/session_coordinator.py +0 -0
  43. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/services/__init__.py +0 -0
  44. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/services/auth_navigation_service.py +0 -0
  45. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/services/field_interaction_service.py +0 -0
  46. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/services/form_filler_service.py +0 -0
  47. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/services/inspection_formatter_service.py +0 -0
  48. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/services/inspection_service.py +0 -0
  49. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/services/reactive_interaction_service.py +0 -0
  50. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/supervisor/__init__.py +0 -0
  51. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/supervisor/client.py +0 -0
  52. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/supervisor/contracts.py +0 -0
  53. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/supervisor/master_daemon.py +0 -0
  54. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/supervisor/security.py +0 -0
  55. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/supervisor/shell.py +0 -0
  56. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/supervisor/worker_process.py +0 -0
  57. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/tests/test_config_and_builder.py +0 -0
  58. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/tests/test_contracts.py +0 -0
  59. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/tests/test_engine_facade.py +0 -0
  60. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/tests/test_enterprise_integration.py +0 -0
  61. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/tests/test_live_session.py +0 -0
  62. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/tests/test_mcp_server.py +0 -0
  63. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/tests/test_orchestration.py +0 -0
  64. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/tests/test_process_manager.py +0 -0
  65. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/tests/test_redaction.py +0 -0
  66. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/tests/test_secrets_resolution.py +0 -0
  67. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/tests/test_services.py +0 -0
  68. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/tests/test_supervisor.py +0 -0
  69. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/tests/test_supervisor_security.py +0 -0
  70. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/validators/__init__.py +0 -0
  71. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/validators/form_validator.py +0 -0
  72. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/validators/payload_validator.py +0 -0
  73. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/webpilot_engine.egg-info/SOURCES.txt +0 -0
  74. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/webpilot_engine.egg-info/dependency_links.txt +0 -0
  75. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/webpilot_engine.egg-info/entry_points.txt +0 -0
  76. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/webpilot_engine.egg-info/requires.txt +0 -0
  77. {webpilot_engine-2.0.5 → webpilot_engine-2.0.6}/webpilot_engine.egg-info/top_level.txt +0 -0
@@ -1,304 +1,304 @@
1
- Metadata-Version: 2.4
2
- Name: webpilot-engine
3
- Version: 2.0.5
4
- Summary: WebPilot: Enterprise Autonomous Browser Engine, Universal Form Inspector & AI Agent Web Runtime
5
- Author: Abdulkarim Salih, WebPilot Core Team
6
- License: MIT
7
- Project-URL: Homepage, https://github.com/AbdulkarimSSS/webpilot
8
- Project-URL: Documentation, https://github.com/AbdulkarimSSS/webpilot#readme
9
- Project-URL: Repository, https://github.com/AbdulkarimSSS/webpilot.git
10
- Project-URL: Bug Tracker, https://github.com/AbdulkarimSSS/webpilot/issues
11
- Project-URL: Changelog, https://github.com/AbdulkarimSSS/webpilot/blob/master/CHANGELOG.md
12
- Keywords: webpilot,browser-runtime,ai-agent,web-automation,playwright,form-filler,ats-automation,sap-successfactors,workday,browser-supervisor,process-isolation
13
- Classifier: Development Status :: 4 - Beta
14
- Classifier: Intended Audience :: Developers
15
- Classifier: Topic :: Software Development :: Testing
16
- Classifier: Topic :: Internet :: WWW/HTTP :: Browsers
17
- Classifier: Programming Language :: Python :: 3
18
- Classifier: Programming Language :: Python :: 3.10
19
- Classifier: Programming Language :: Python :: 3.11
20
- Classifier: Programming Language :: Python :: 3.12
21
- Classifier: Programming Language :: Python :: 3.13
22
- Classifier: Operating System :: Microsoft :: Windows
23
- Classifier: Operating System :: POSIX :: Linux
24
- Classifier: Operating System :: MacOS
25
- Requires-Python: >=3.10
26
- Description-Content-Type: text/markdown
27
- Requires-Dist: playwright>=1.40.0
28
- Requires-Dist: python-dotenv>=1.0.0
29
- Provides-Extra: dev
30
- Requires-Dist: pytest>=8.0.0; extra == "dev"
31
- Requires-Dist: pytest-asyncio>=0.23.0; extra == "dev"
32
- Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
33
- Requires-Dist: pytest-mock>=3.12.0; extra == "dev"
34
- Requires-Dist: Faker>=24.0.0; extra == "dev"
35
- Provides-Extra: mcp
36
- Requires-Dist: mcp<2,>=1.0.0; extra == "mcp"
37
-
38
- # WebPilot: Autonomous Multi-Tier Browser Automation Engine & AI Agent Web Runtime
39
-
40
- [![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/)
41
- [![Playwright](https://img.shields.io/badge/playwright-v1.40%2B-green.svg)](https://playwright.dev/)
42
- [![Architecture](https://img.shields.io/badge/architecture-3--Tier%20Multi--Process-orange.svg)](#-3-tier-multi-process-architecture)
43
- [![Standard User](https://img.shields.io/badge/security-Zero%20Elevation%20(No%20Admin)-success.svg)](#-zero-elevation-standard-user-security)
44
- [![Tests Passing](https://img.shields.io/badge/tests-27%2F27%20passing-brightgreen.svg)](#-testing--verification)
45
- [![CLI](https://img.shields.io/badge/cli-webpilot%20%7C%20wp-6f42c1.svg)](#-quick-start-guide)
46
-
47
- An enterprise-grade, general-purpose autonomous browser engine and AI agent web runtime designed to inspect, interact with, fill, and operate any website, complex Single-Page Application (SPA), or enterprise portal (such as **SAP SuccessFactors**, **Workday**, **Greenhouse**, **Oracle Taleo**, and modern web apps) with zero site-specific selectors or hardcoding.
48
-
49
- ---
50
-
51
- ## 🏗️ 3-Tier Multi-Process Architecture
52
-
53
- To solve the fundamental Windows OS limitation where terminal and subshell runners terminate background child processes upon exit (Windows Job Object `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`), WebPilot implements an elevated-resilient **Zero-Elevation Three-Tier Process Architecture**:
54
-
55
- ```
56
- ┌──────────────────────────────────────────────────────────────┐
57
- │ WebPilot CLI & REPL Shell (webpilot / wp) │
58
- └──────────────────────────────┬───────────────────────────────┘
59
- │ HTTP REST (127.0.0.1:9333)
60
- ▼
61
- ┌──────────────────────────────────────────────────────────────┐
62
- │ Layer 1: Master Supervisor Host (Daemon) │
63
- │ • Zero Playwright Imports (~15 MB RAM) │
64
- │ • 30-Minute Inactivity Watchdog (Automatic RAM release) │
65
- │ • Windows WMI Zero-Touch Auto-Spawn (Session 0 Breakout) │
66
- └──────────────────────────────┬───────────────────────────────┘
67
- │ stdin / stdout (Isolated JSON-RPC)
68
- ▼
69
- ┌──────────────────────────────────────────────────────────────┐
70
- │ Layer 2: Operational Worker Process │
71
- │ • Playwright Runtime + SessionCoordinator │
72
- │ • Persistent in-memory browser session across CLI calls │
73
- │ • DOM Services (Field, Auth, Reactive, Inspection) │
74
- └──────────────────────────────┬───────────────────────────────┘
75
- │ Process Tree Attachment
76
- ▼
77
- ┌──────────────────────────────────────────────────────────────┐
78
- │ Layer 3: Chromium Browser Engine │
79
- │ • Chromium Main Process + GPU + Renderers │
80
- │ • Immediate Cascading Kill (taskkill /F /T /PID) │
81
- └──────────────────────────────────────────────────────────────┘
82
- ```
83
-
84
- ### ⚡ Key Architectural Advantages:
85
- 1. **Persistent Browser Session Across CLI Calls**: Run step-by-step sequential commands (`open` -> `fill username` -> `fill password` -> `press submit`) without restarting the browser or losing active DOM state.
86
- 2. **Zero-Elevation / Standard User Mode**: Runs 100% as a standard, unprivileged user. **Zero administrative or UAC prompts required** (no `asudo` or Administrator escalation).
87
- 3. **Cascading Kill & Instant Self-Healing**: Terminating Layer 2 via `taskkill /F /T /PID` instantly eliminates all child Chromium processes with **zero zombie leaks**. Layer 1 survives and automatically respawns a clean Layer 2 in **~300ms**.
88
- 4. **30-Minute Watchdog Supervisor**: Automatically reclaims 100% of browser memory if the session remains idle for 30 minutes.
89
-
90
- ---
91
-
92
- ## 🌟 Core Features
93
-
94
- - **Universal Inspection (`inspect`)**: Navigates to any URL and outputs a token-optimized, clean schema containing text fields, picklists, radio groups, checkboxes, file uploads, and action buttons.
95
- - **Dynamic Field Population (`apply`)**: Fills fields dynamically via JSON payloads (`--data`) or inline CLI flags (`--fill`), with automatic type detection (text, combobox, radio, select).
96
- - **Reactive Interaction Service**:
97
- - Automatically observes button clicks (`--press`).
98
- - Detects redirects and route transitions (`[🔀 Redirect]`) and auto-inspects the resulting page.
99
- - Detects and switches to newly spawned browser tabs/windows (`[🌐 New Window]`).
100
- - Detects dynamic DOM expansions and section accordions (`[📋 Dynamic Form Expansion]`).
101
- - **Dynamic Loader Dissolution**:
102
- - Automatically identifies global blocking overlays (`#loading`, `.sapUiBusy`, `.busyIndicator`, `.modal-backdrop`, `[aria-busy="true"]`) and waits dynamically for their dissolution (`state="hidden"`).
103
- - Distinguishes between global page-blocking veils and local dropdown loaders.
104
- - **Modal & Pointer Trap Escape Pipeline**:
105
- - Resolves modal popups and pointer-locked elements using a 4-stage pipeline: Natural Selection -> Keyboard `Escape` -> Backdrop Click -> DOM Surgical Removal.
106
- - **Enterprise Picklist Auto-scrolling**:
107
- - Intelligent scrolling algorithm for virtualized dropdowns (such as SAP picklists with `aria-owns` and scroll containers).
108
- - **Interactive REPL Shell (`shell`)**:
109
- - Interactive automation console with real-time feedback, tab switching, and live DOM inspection.
110
-
111
- ---
112
-
113
- ## 📦 Installation & Setup
114
-
115
- ### Option A: Install from PyPI or Editable Mode
116
- ```bash
117
- # Editable install into environment
118
- pip install -e .
119
-
120
- # Or install dependencies directly
121
- pip install -r requirements.txt
122
- playwright install chromium
123
- ```
124
-
125
- ### Option B: Standalone Portable Binary (Windows)
126
- Download `webpilot.exe` from GitHub Releases and run directly without needing a local Python installation.
127
-
128
- ---
129
-
130
- ## 🚀 Quick Start Guide
131
-
132
- You can use the full `webpilot` command or the ultra-fast shortcut **`wp`** (or `python cli.py`):
133
-
134
- ### 1. Check & Start Background Supervisor Service
135
- ```powershell
136
- # Check multi-tier service status
137
- wp service status
138
-
139
- # Start service (automatically auto-spawns detached host via WMI)
140
- wp service start
141
- ```
142
-
143
- ### 2. Inspect Any Web Page
144
- ```powershell
145
- wp inspect --url "https://example.com/portal" --output "schema.json" --screenshot "page.png"
146
- ```
147
-
148
- ### 3. Step-by-Step Multi-Turn Web Automation
149
- Execute sequential commands on the **same live persistent browser session**:
150
-
151
- ```powershell
152
- # Step 1: Open target portal and inspect initial schema
153
- wp apply --url "https://example.com/login" --screenshot "step1.png"
154
-
155
- # Step 2: Fill email only on the active persistent page
156
- wp apply --fill "username=john.doe@example.com" --screenshot "step2.png"
157
-
158
- # Step 3: Fill password only on the active persistent page
159
- wp apply --fill "password=MySecurePassword123!" --screenshot "step3.png"
160
-
161
- # Step 4: Click Sign In, detect redirect, auto-inspect profile, and save cookies
162
- wp apply --press "Sign In" --cookies "session_cookies.json" --screenshot "step4.png"
163
- ```
164
-
165
- ### 4. Interactive REPL Shell Mode
166
- Launch the sub-second interactive REPL shell:
167
- ```powershell
168
- wp shell
169
- ```
170
-
171
- Inside the shell, enter commands sequentially:
172
- ```text
173
- webpilot> open https://example.com/login
174
- webpilot> fill username=john.doe@example.com
175
- webpilot> fill password=MySecurePassword123!
176
- webpilot> press Sign In
177
- webpilot> tabs
178
- webpilot> screenshot dashboard.png
179
- webpilot> exit
180
- ```
181
-
182
- ---
183
-
184
- ## 🛠️ CLI Reference
185
-
186
- ### Common Global Flags
187
- - `--host`: Supervisor host address (default: `127.0.0.1`).
188
- - `--port`: Supervisor port (default: `9333`).
189
- - `--standalone`: Bypasses Master Supervisor to run in-process via `FlowOrchestrator`.
190
-
191
- ### `python cli.py inspect`
192
- Inspects a URL or active session and outputs a clean form schema:
193
- - `--url <URL>`: Target URL (optional if connecting to existing live session).
194
- - `--output <path>`: Save full JSON schema to disk.
195
- - `--screenshot <path>`: Capture page screenshot.
196
- - `--cookies <path>`: Load session cookies.
197
- - `--unpack-options`: Probe dropdowns to sample options.
198
- - `--tab <index>`: Inspect specific tab by index.
199
- - `--list-tabs`: List all open tabs in active session.
200
-
201
- ### `python cli.py apply`
202
- Fills fields, uploads files, presses buttons, and validates state:
203
- - `--url <URL>`: Target URL (optional if operating on existing page).
204
- - `--data <path>`: Load key-value mappings from JSON file.
205
- - `--fill <key=val>`: Set field value (can be repeated).
206
- - `--press <btn>`: Click button with reactive observation (can be repeated).
207
- - `--upload <kw=file>`: Upload document to upload area (can be repeated).
208
- - `--cookies <path>`: Save or load session cookies.
209
- - `--submit`: Submit form after filling.
210
- - `--screenshot <path>`: Capture post-interaction screenshot.
211
- - `--no-auto-inspect`: Disable automatic post-interaction schema inspection.
212
-
213
- ### `python cli.py service`
214
- Manages the Master Supervisor daemon:
215
- - `python cli.py service status`: Query real-time PID telemetry for Layer 1, Layer 2, and Layer 3.
216
- - `python cli.py service restart`: Trigger cascading kill of Layer 2 & Layer 3, and respawn fresh worker (~300ms).
217
- - `python cli.py service stop`: Cleanly terminate all tiers and release all memory.
218
- - `python cli.py service start`: Ensure Layer 1 daemon is running.
219
-
220
- ---
221
-
222
- ## 📁 Project Directory Layout
223
-
224
- ```
225
- universal_web_applier/
226
- ├── adapters/
227
- │ ├── browser_adapter.py # Playwright & Chromium connection adapter
228
- │ ├── dom_scripts.py # Pure JavaScript DOM manipulation scripts
229
- │ ├── live_session_manager.py # Legacy CDP live session fallback
230
- │ └── upload_adapter.py # Intelligent file upload handler
231
- ├── config/
232
- │ ├── builder.py # Payload & Envelope builders
233
- │ ├── settings.py # 3-tier settings resolver (env -> .env -> json)
234
- │ ├── settings.json # Active settings
235
- │ └── device_profile.json # Browser fingerprint & viewport settings
236
- ├── constants/
237
- │ ├── contracts.py # Single Source of Truth for JSON contracts
238
- │ └── timeouts.py # Centralized timeouts and pauses
239
- ├── core/
240
- │ └── models.py # Dataclasses & schema models
241
- ├── orchestration/
242
- │ ├── context.py # Request & session contexts
243
- │ ├── flow_orchestrator.py # In-process standalone workflow pipeline
244
- │ └── session_coordinator.py # Session lifecycle coordinator
245
- ├── services/
246
- │ ├── auth_navigation_service.py # Login detection & portal transitions
247
- │ ├── field_interaction_service.py# Field value injection & verification
248
- │ ├── inspection_service.py # DOM inspection & schema extraction
249
- │ ├── inspection_formatter_service.py # Lean human-readable summary formatter
250
- │ └── reactive_interaction_service.py # Post-click reactive changes observer
251
- ├── supervisor/
252
- │ ├── contracts.py # Supervisor request/response contracts
253
- │ ├── master_daemon.py # Layer 1: Lightweight HTTP Daemon
254
- │ ├── worker_process.py # Layer 2: Supervised JSON-RPC Worker
255
- │ ├── client.py # Zero-Touch WMI client & HTTP proxy
256
- │ └── shell.py # Interactive REPL automation shell
257
- ├── tests/
258
- │ ├── test_contracts.py # Wire contract unit tests
259
- │ ├── test_engine_facade.py # Engine facade tests
260
- │ ├── test_live_session.py # Live session tests
261
- │ ├── test_orchestration.py # Pipeline orchestration tests
262
- │ ├── test_redaction.py # Security & credential redaction tests
263
- │ ├── test_services.py # DOM interaction services tests
264
- │ └── test_supervisor.py # Master supervisor & client unit tests
265
- ├── cli.py # Unified CLI entrypoint
266
- ├── requirements.txt # Python dependencies
267
- ├── pyproject.toml # Modern package specifications
268
- └── README.md # Master documentation
269
- ```
270
-
271
- ---
272
-
273
- ## 🧪 Testing & Verification
274
-
275
- Run the comprehensive unit test suite:
276
- ```powershell
277
- python -m pytest tests/
278
- ```
279
-
280
- All 27 contract and service tests execute in < 25s:
281
- ```text
282
- tests\test_contracts.py .... [ 14%]
283
- tests\test_engine_facade.py ... [ 25%]
284
- tests\test_live_session.py .. [ 33%]
285
- tests\test_orchestration.py ... [ 44%]
286
- tests\test_redaction.py .... [ 59%]
287
- tests\test_services.py ....... [ 85%]
288
- tests\test_supervisor.py .... [100%]
289
-
290
- ============================= 27 passed in 22.58s =============================
291
- ```
292
-
293
- ---
294
-
295
- ## 🔒 Security & Privacy
296
-
297
- - **Credential Redaction**: Passwords, auth tokens, and sensitive inputs are automatically redacted in CLI outputs, telemetry cards, and logs.
298
- - **Zero Administrative Elevation**: Does not require Administrator rights or elevated Windows tokens.
299
- - **Local Isolation**: Master Supervisor listens exclusively on loopback `127.0.0.1:9333`.
300
-
301
- ---
302
-
303
- ## 📄 License
304
- MIT License. Free for commercial and personal use.
1
+ Metadata-Version: 2.4
2
+ Name: webpilot-engine
3
+ Version: 2.0.6
4
+ Summary: WebPilot: Enterprise Autonomous Browser Engine, Universal Form Inspector & AI Agent Web Runtime
5
+ Author: Abdulkarim Salih, WebPilot Core Team
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/AbdulkarimSSS/webpilot
8
+ Project-URL: Documentation, https://github.com/AbdulkarimSSS/webpilot#readme
9
+ Project-URL: Repository, https://github.com/AbdulkarimSSS/webpilot.git
10
+ Project-URL: Bug Tracker, https://github.com/AbdulkarimSSS/webpilot/issues
11
+ Project-URL: Changelog, https://github.com/AbdulkarimSSS/webpilot/blob/master/CHANGELOG.md
12
+ Keywords: webpilot,browser-runtime,ai-agent,web-automation,playwright,form-filler,ats-automation,sap-successfactors,workday,browser-supervisor,process-isolation
13
+ Classifier: Development Status :: 4 - Beta
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: Topic :: Software Development :: Testing
16
+ Classifier: Topic :: Internet :: WWW/HTTP :: Browsers
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Programming Language :: Python :: 3.10
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Programming Language :: Python :: 3.13
22
+ Classifier: Operating System :: Microsoft :: Windows
23
+ Classifier: Operating System :: POSIX :: Linux
24
+ Classifier: Operating System :: MacOS
25
+ Requires-Python: >=3.10
26
+ Description-Content-Type: text/markdown
27
+ Requires-Dist: playwright>=1.40.0
28
+ Requires-Dist: python-dotenv>=1.0.0
29
+ Provides-Extra: dev
30
+ Requires-Dist: pytest>=8.0.0; extra == "dev"
31
+ Requires-Dist: pytest-asyncio>=0.23.0; extra == "dev"
32
+ Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
33
+ Requires-Dist: pytest-mock>=3.12.0; extra == "dev"
34
+ Requires-Dist: Faker>=24.0.0; extra == "dev"
35
+ Provides-Extra: mcp
36
+ Requires-Dist: mcp<2,>=1.0.0; extra == "mcp"
37
+
38
+ # WebPilot: Autonomous Multi-Tier Browser Automation Engine & AI Agent Web Runtime
39
+
40
+ [![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/)
41
+ [![Playwright](https://img.shields.io/badge/playwright-v1.40%2B-green.svg)](https://playwright.dev/)
42
+ [![Architecture](https://img.shields.io/badge/architecture-3--Tier%20Multi--Process-orange.svg)](#-3-tier-multi-process-architecture)
43
+ [![Standard User](https://img.shields.io/badge/security-Zero%20Elevation%20(No%20Admin)-success.svg)](#-zero-elevation-standard-user-security)
44
+ [![Tests Passing](https://img.shields.io/badge/tests-27%2F27%20passing-brightgreen.svg)](#-testing--verification)
45
+ [![CLI](https://img.shields.io/badge/cli-webpilot%20%7C%20wp-6f42c1.svg)](#-quick-start-guide)
46
+
47
+ An enterprise-grade, general-purpose autonomous browser engine and AI agent web runtime designed to inspect, interact with, fill, and operate any website, complex Single-Page Application (SPA), or enterprise portal (such as **SAP SuccessFactors**, **Workday**, **Greenhouse**, **Oracle Taleo**, and modern web apps) with zero site-specific selectors or hardcoding.
48
+
49
+ ---
50
+
51
+ ## 🏗️ 3-Tier Multi-Process Architecture
52
+
53
+ To solve the fundamental Windows OS limitation where terminal and subshell runners terminate background child processes upon exit (Windows Job Object `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`), WebPilot implements an elevated-resilient **Zero-Elevation Three-Tier Process Architecture**:
54
+
55
+ ```
56
+ ┌──────────────────────────────────────────────────────────────┐
57
+ │ WebPilot CLI & REPL Shell (webpilot / wp) │
58
+ └──────────────────────────────┬───────────────────────────────┘
59
+ │ HTTP REST (127.0.0.1:9333)
60
+ ▼
61
+ ┌──────────────────────────────────────────────────────────────┐
62
+ │ Layer 1: Master Supervisor Host (Daemon) │
63
+ │ • Zero Playwright Imports (~15 MB RAM) │
64
+ │ • 30-Minute Inactivity Watchdog (Automatic RAM release) │
65
+ │ • Windows WMI Zero-Touch Auto-Spawn (Session 0 Breakout) │
66
+ └──────────────────────────────┬───────────────────────────────┘
67
+ │ stdin / stdout (Isolated JSON-RPC)
68
+ ▼
69
+ ┌──────────────────────────────────────────────────────────────┐
70
+ │ Layer 2: Operational Worker Process │
71
+ │ • Playwright Runtime + SessionCoordinator │
72
+ │ • Persistent in-memory browser session across CLI calls │
73
+ │ • DOM Services (Field, Auth, Reactive, Inspection) │
74
+ └──────────────────────────────┬───────────────────────────────┘
75
+ │ Process Tree Attachment
76
+ ▼
77
+ ┌──────────────────────────────────────────────────────────────┐
78
+ │ Layer 3: Chromium Browser Engine │
79
+ │ • Chromium Main Process + GPU + Renderers │
80
+ │ • Immediate Cascading Kill (taskkill /F /T /PID) │
81
+ └──────────────────────────────────────────────────────────────┘
82
+ ```
83
+
84
+ ### ⚡ Key Architectural Advantages:
85
+ 1. **Persistent Browser Session Across CLI Calls**: Run step-by-step sequential commands (`open` -> `fill username` -> `fill password` -> `press submit`) without restarting the browser or losing active DOM state.
86
+ 2. **Zero-Elevation / Standard User Mode**: Runs 100% as a standard, unprivileged user. **Zero administrative or UAC prompts required** (no `asudo` or Administrator escalation).
87
+ 3. **Cascading Kill & Instant Self-Healing**: Terminating Layer 2 via `taskkill /F /T /PID` instantly eliminates all child Chromium processes with **zero zombie leaks**. Layer 1 survives and automatically respawns a clean Layer 2 in **~300ms**.
88
+ 4. **30-Minute Watchdog Supervisor**: Automatically reclaims 100% of browser memory if the session remains idle for 30 minutes.
89
+
90
+ ---
91
+
92
+ ## 🌟 Core Features
93
+
94
+ - **Universal Inspection (`inspect`)**: Navigates to any URL and outputs a token-optimized, clean schema containing text fields, picklists, radio groups, checkboxes, file uploads, and action buttons.
95
+ - **Dynamic Field Population (`apply`)**: Fills fields dynamically via JSON payloads (`--data`) or inline CLI flags (`--fill`), with automatic type detection (text, combobox, radio, select).
96
+ - **Reactive Interaction Service**:
97
+ - Automatically observes button clicks (`--press`).
98
+ - Detects redirects and route transitions (`[🔀 Redirect]`) and auto-inspects the resulting page.
99
+ - Detects and switches to newly spawned browser tabs/windows (`[🌐 New Window]`).
100
+ - Detects dynamic DOM expansions and section accordions (`[📋 Dynamic Form Expansion]`).
101
+ - **Dynamic Loader Dissolution**:
102
+ - Automatically identifies global blocking overlays (`#loading`, `.sapUiBusy`, `.busyIndicator`, `.modal-backdrop`, `[aria-busy="true"]`) and waits dynamically for their dissolution (`state="hidden"`).
103
+ - Distinguishes between global page-blocking veils and local dropdown loaders.
104
+ - **Modal & Reactive Interaction Pipeline**:
105
+ - Resolves modal popups and interactive dialogs using a 4-strategy non-destructive cascade: Native ID -> ARIA Role/Text Cascade -> JS In-DOM Click -> W3C `Escape` Key Protocol, preserving full DOM integrity.
106
+ - **Enterprise Picklist Auto-scrolling**:
107
+ - Intelligent scrolling algorithm for virtualized dropdowns (such as SAP picklists with `aria-owns` and scroll containers).
108
+ - **Interactive REPL Shell (`shell`)**:
109
+ - Interactive automation console with real-time feedback, tab switching, and live DOM inspection.
110
+
111
+ ---
112
+
113
+ ## 📦 Installation & Setup
114
+
115
+ ### Option A: Install from PyPI or Editable Mode
116
+ ```bash
117
+ # Editable install into environment
118
+ pip install -e .
119
+
120
+ # Or install dependencies directly
121
+ pip install -r requirements.txt
122
+ playwright install chromium
123
+ ```
124
+
125
+ ### Option B: Standalone Portable Binary (Windows)
126
+ Download `webpilot.exe` from GitHub Releases and run directly without needing a local Python installation.
127
+
128
+ ---
129
+
130
+ ## 🚀 Quick Start Guide
131
+
132
+ You can use the full `webpilot` command or the ultra-fast shortcut **`wp`** (or `python cli.py`):
133
+
134
+ ### 1. Check & Start Background Supervisor Service
135
+ ```powershell
136
+ # Check multi-tier service status
137
+ wp service status
138
+
139
+ # Start service (automatically auto-spawns detached host via WMI)
140
+ wp service start
141
+ ```
142
+
143
+ ### 2. Inspect Any Web Page
144
+ ```powershell
145
+ wp inspect --url "https://example.com/portal" --output "schema.json" --screenshot "page.png"
146
+ ```
147
+
148
+ ### 3. Step-by-Step Multi-Turn Web Automation
149
+ Execute sequential commands on the **same live persistent browser session**:
150
+
151
+ ```powershell
152
+ # Step 1: Open target portal and inspect initial schema
153
+ wp apply --url "https://example.com/login" --screenshot "step1.png"
154
+
155
+ # Step 2: Fill email only on the active persistent page
156
+ wp apply --fill "username=john.doe@example.com" --screenshot "step2.png"
157
+
158
+ # Step 3: Fill password only on the active persistent page
159
+ wp apply --fill "password=MySecurePassword123!" --screenshot "step3.png"
160
+
161
+ # Step 4: Click Sign In, detect redirect, auto-inspect profile, and save cookies
162
+ wp apply --press "Sign In" --cookies "session_cookies.json" --screenshot "step4.png"
163
+ ```
164
+
165
+ ### 4. Interactive REPL Shell Mode
166
+ Launch the sub-second interactive REPL shell:
167
+ ```powershell
168
+ wp shell
169
+ ```
170
+
171
+ Inside the shell, enter commands sequentially:
172
+ ```text
173
+ webpilot> open https://example.com/login
174
+ webpilot> fill username=john.doe@example.com
175
+ webpilot> fill password=MySecurePassword123!
176
+ webpilot> press Sign In
177
+ webpilot> tabs
178
+ webpilot> screenshot dashboard.png
179
+ webpilot> exit
180
+ ```
181
+
182
+ ---
183
+
184
+ ## 🛠️ CLI Reference
185
+
186
+ ### Common Global Flags
187
+ - `--host`: Supervisor host address (default: `127.0.0.1`).
188
+ - `--port`: Supervisor port (default: `9333`).
189
+ - `--standalone`: Bypasses Master Supervisor to run in-process via `FlowOrchestrator`.
190
+
191
+ ### `python cli.py inspect`
192
+ Inspects a URL or active session and outputs a clean form schema:
193
+ - `--url <URL>`: Target URL (optional if connecting to existing live session).
194
+ - `--output <path>`: Save full JSON schema to disk.
195
+ - `--screenshot <path>`: Capture page screenshot.
196
+ - `--cookies <path>`: Load session cookies.
197
+ - `--unpack-options`: Probe dropdowns to sample options.
198
+ - `--tab <index>`: Inspect specific tab by index.
199
+ - `--list-tabs`: List all open tabs in active session.
200
+
201
+ ### `python cli.py apply`
202
+ Fills fields, uploads files, presses buttons, and validates state:
203
+ - `--url <URL>`: Target URL (optional if operating on existing page).
204
+ - `--data <path>`: Load key-value mappings from JSON file.
205
+ - `--fill <key=val>`: Set field value (can be repeated).
206
+ - `--press <btn>`: Click button with reactive observation (can be repeated).
207
+ - `--upload <kw=file>`: Upload document to upload area (can be repeated).
208
+ - `--cookies <path>`: Save or load session cookies.
209
+ - `--submit`: Submit form after filling.
210
+ - `--screenshot <path>`: Capture post-interaction screenshot.
211
+ - `--no-auto-inspect`: Disable automatic post-interaction schema inspection.
212
+
213
+ ### `python cli.py service`
214
+ Manages the Master Supervisor daemon:
215
+ - `python cli.py service status`: Query real-time PID telemetry for Layer 1, Layer 2, and Layer 3.
216
+ - `python cli.py service restart`: Trigger cascading kill of Layer 2 & Layer 3, and respawn fresh worker (~300ms).
217
+ - `python cli.py service stop`: Cleanly terminate all tiers and release all memory.
218
+ - `python cli.py service start`: Ensure Layer 1 daemon is running.
219
+
220
+ ---
221
+
222
+ ## 📁 Project Directory Layout
223
+
224
+ ```
225
+ universal_web_applier/
226
+ ├── adapters/
227
+ │ ├── browser_adapter.py # Playwright & Chromium connection adapter
228
+ │ ├── dom_scripts.py # Pure JavaScript DOM manipulation scripts
229
+ │ ├── live_session_manager.py # Legacy CDP live session fallback
230
+ │ └── upload_adapter.py # Intelligent file upload handler
231
+ ├── config/
232
+ │ ├── builder.py # Payload & Envelope builders
233
+ │ ├── settings.py # 3-tier settings resolver (env -> .env -> json)
234
+ │ ├── settings.json # Active settings
235
+ │ └── device_profile.json # Browser fingerprint & viewport settings
236
+ ├── constants/
237
+ │ ├── contracts.py # Single Source of Truth for JSON contracts
238
+ │ └── timeouts.py # Centralized timeouts and pauses
239
+ ├── core/
240
+ │ └── models.py # Dataclasses & schema models
241
+ ├── orchestration/
242
+ │ ├── context.py # Request & session contexts
243
+ │ ├── flow_orchestrator.py # In-process standalone workflow pipeline
244
+ │ └── session_coordinator.py # Session lifecycle coordinator
245
+ ├── services/
246
+ │ ├── auth_navigation_service.py # Login detection & portal transitions
247
+ │ ├── field_interaction_service.py# Field value injection & verification
248
+ │ ├── inspection_service.py # DOM inspection & schema extraction
249
+ │ ├── inspection_formatter_service.py # Lean human-readable summary formatter
250
+ │ └── reactive_interaction_service.py # Post-click reactive changes observer
251
+ ├── supervisor/
252
+ │ ├── contracts.py # Supervisor request/response contracts
253
+ │ ├── master_daemon.py # Layer 1: Lightweight HTTP Daemon
254
+ │ ├── worker_process.py # Layer 2: Supervised JSON-RPC Worker
255
+ │ ├── client.py # Zero-Touch WMI client & HTTP proxy
256
+ │ └── shell.py # Interactive REPL automation shell
257
+ ├── tests/
258
+ │ ├── test_contracts.py # Wire contract unit tests
259
+ │ ├── test_engine_facade.py # Engine facade tests
260
+ │ ├── test_live_session.py # Live session tests
261
+ │ ├── test_orchestration.py # Pipeline orchestration tests
262
+ │ ├── test_redaction.py # Security & credential redaction tests
263
+ │ ├── test_services.py # DOM interaction services tests
264
+ │ └── test_supervisor.py # Master supervisor & client unit tests
265
+ ├── cli.py # Unified CLI entrypoint
266
+ ├── requirements.txt # Python dependencies
267
+ ├── pyproject.toml # Modern package specifications
268
+ └── README.md # Master documentation
269
+ ```
270
+
271
+ ---
272
+
273
+ ## 🧪 Testing & Verification
274
+
275
+ Run the comprehensive unit test suite:
276
+ ```powershell
277
+ python -m pytest tests/
278
+ ```
279
+
280
+ All 27 contract and service tests execute in < 25s:
281
+ ```text
282
+ tests\test_contracts.py .... [ 14%]
283
+ tests\test_engine_facade.py ... [ 25%]
284
+ tests\test_live_session.py .. [ 33%]
285
+ tests\test_orchestration.py ... [ 44%]
286
+ tests\test_redaction.py .... [ 59%]
287
+ tests\test_services.py ....... [ 85%]
288
+ tests\test_supervisor.py .... [100%]
289
+
290
+ ============================= 27 passed in 22.58s =============================
291
+ ```
292
+
293
+ ---
294
+
295
+ ## 🔒 Security & Privacy
296
+
297
+ - **Credential Redaction**: Passwords, auth tokens, and sensitive inputs are automatically redacted in CLI outputs, telemetry cards, and logs.
298
+ - **Zero Administrative Elevation**: Does not require Administrator rights or elevated Windows tokens.
299
+ - **Local Isolation**: Master Supervisor listens exclusively on loopback `127.0.0.1:9333`.
300
+
301
+ ---
302
+
303
+ ## 📄 License
304
+ MIT License. Free for commercial and personal use.
@@ -64,8 +64,8 @@ To solve the fundamental Windows OS limitation where terminal and subshell runne
64
64
  - **Dynamic Loader Dissolution**:
65
65
  - Automatically identifies global blocking overlays (`#loading`, `.sapUiBusy`, `.busyIndicator`, `.modal-backdrop`, `[aria-busy="true"]`) and waits dynamically for their dissolution (`state="hidden"`).
66
66
  - Distinguishes between global page-blocking veils and local dropdown loaders.
67
- - **Modal & Pointer Trap Escape Pipeline**:
68
- - Resolves modal popups and pointer-locked elements using a 4-stage pipeline: Natural Selection -> Keyboard `Escape` -> Backdrop Click -> DOM Surgical Removal.
67
+ - **Modal & Reactive Interaction Pipeline**:
68
+ - Resolves modal popups and interactive dialogs using a 4-strategy non-destructive cascade: Native ID -> ARIA Role/Text Cascade -> JS In-DOM Click -> W3C `Escape` Key Protocol, preserving full DOM integrity.
69
69
  - **Enterprise Picklist Auto-scrolling**:
70
70
  - Intelligent scrolling algorithm for virtualized dropdowns (such as SAP picklists with `aria-owns` and scroll containers).
71
71
  - **Interactive REPL Shell (`shell`)**: