webtap-tool 0.7.0__py3-none-any.whl → 0.8.0__py3-none-any.whl

webtap/commands/TIPS.md

@@ -12,13 +12,21 @@ All commands have these pre-imported (no imports needed!):
 ## Commands
 
 ### body
-Fetch and analyze HTTP response bodies with Python expressions.
+Fetch and analyze HTTP request or response bodies with Python expressions. Automatically detects event type.
 
 #### Examples
 ```python
-body(123)                                  # Get body
-body(123, "json.loads(body)")              # Parse JSON
+# Response bodies
+body(123)                                  # Get response body
+body(123, "json.loads(body)")              # Parse JSON response
 body(123, "bs4(body, 'html.parser').find('title').text")  # HTML title
+
+# Request bodies (POST/PUT/PATCH)
+body(456)                                  # Get request POST data
+body(456, "json.loads(body)")              # Parse JSON request body
+body(456, "json.loads(body)['customerId']")  # Extract request field
+
+# Analysis
 body(123, "jwt.decode(body, options={'verify_signature': False})")  # Decode JWT
 body(123, "re.findall(r'/api/[^\"\\s]+', body)[:10]")  # Find API endpoints
 body(123, "httpx.get(json.loads(body)['next_url']).json()")  # Chain requests
@@ -26,7 +34,10 @@ body(123, "msgpack.unpackb(body)")         # Binary formats
 ```
 
 #### Tips
+- **Auto-detect type:** Command automatically detects request vs response events
+- **Find request events:** `events({"method": "Network.requestWillBeSent", "url": "*WsJobCard/Post*"})` - POST requests
 - **Generate models:** `to_model({id}, "models/model.py", "Model")` - create Pydantic models from JSON
+- **Generate types:** `quicktype({id}, "types.ts", "Type")` - TypeScript/other languages
 - **Chain requests:** `body({id}, "httpx.get(json.loads(body)['next_url']).text[:100]")`
 - **Parse XML:** `body({id}, "ElementTree.fromstring(body).find('.//title').text")`
 - **Extract forms:** `body({id}, "[f['action'] for f in bs4(body, 'html.parser').find_all('form')]")`
@@ -34,22 +45,62 @@ body(123, "msgpack.unpackb(body)")         # Binary formats
 - **Find related:** `events({'requestId': request_id})` - related events
 
 ### to_model
-Generate Pydantic v2 models from JSON response bodies for reverse engineering APIs.
+Generate Pydantic v2 models from request or response bodies.
 
 #### Examples
 ```python
-to_model(123, "models/product.py", "Product")                              # Generate from full response
-to_model(123, "models/customers/group.py", "CustomerGroup", "Data[0]")     # Extract nested + domain structure
-to_model(123, "/tmp/item.py", "Item", "items[0]")                          # Extract array items
+# Response bodies
+to_model(123, "models/product.py", "Product")                              # Full response
+to_model(123, "models/customers/group.py", "CustomerGroup", json_path="data[0]")  # Extract nested
+
+# Request bodies (POST/PUT/PATCH)
+to_model(172, "models/form.py", "JobCardForm", expr="dict(urllib.parse.parse_qsl(body))")  # Form data
+to_model(180, "models/request.py", "CreateOrder")                          # JSON POST body
+
+# Advanced transformations
+to_model(123, "models/clean.py", "Clean", expr="{k: v for k, v in json.loads(body).items() if k != 'meta'}")
+to_model(123, "models/merged.py", "Merged", expr="{**json.loads(body), 'url': event['params']['response']['url']}")
 ```
 
 #### Tips
-- **Check structure first:** `body({id})` - preview JSON before generating
-- **Domain organization:** Use paths like `"models/customers/group.py"` for structure
-- **Extract nested data:** Use `json_path="Data[0]"` to extract specific objects
-- **Array items:** Extract first item with `json_path="items[0]"` for model generation
-- **Auto-cleanup:** Generated models use snake_case fields and modern type hints (list, dict, | None)
-- **Edit after:** Add `Field(alias="...")` manually for API field mapping
+- **Check structure:** `body({id})` - preview body before generating
+- **Find requests:** `events({"method": "Network.requestWillBeSent", "url": "*api/orders*"})` - locate POST events
+- **Form data:** `expr="dict(urllib.parse.parse_qsl(body))"` for application/x-www-form-urlencoded
+- **Nested extraction:** `json_path="data[0]"` for JSON with wrapper objects
+- **Custom transforms:** `expr` has `body` (str) and `event` (dict) variables available
+- **Organization:** Paths like `"models/customers/group.py"` create directory structure automatically
+- **Field mapping:** Add `Field(alias="...")` after generation for API field names
+
+### quicktype
+Generate types from request or response bodies. Supports TypeScript, Go, Rust, Python, and 10+ other languages.
+
+#### Examples
+```python
+# Response bodies
+quicktype(123, "types/User.ts", "User")                                    # TypeScript
+quicktype(123, "api.go", "ApiResponse")                                    # Go struct
+quicktype(123, "schema.json", "Schema")                                    # JSON Schema
+quicktype(123, "types.ts", "User", json_path="data[0]")                    # Extract nested
+
+# Request bodies (POST/PUT/PATCH)
+quicktype(172, "types/JobCard.ts", "JobCardForm", expr="dict(urllib.parse.parse_qsl(body))")  # Form data
+quicktype(180, "types/CreateOrder.ts", "CreateOrderRequest")               # JSON POST body
+
+# Advanced options
+quicktype(123, "types.ts", "User", options={"readonly": True})             # TypeScript readonly
+quicktype(123, "types.ts", "Clean", expr="{k: v for k, v in json.loads(body).items() if k != 'meta'}")
+```
+
+#### Tips
+- **Check structure:** `body({id})` - preview body before generating
+- **Find requests:** `events({"method": "Network.requestWillBeSent", "url": "*api*"})` - locate POST events
+- **Form data:** `expr="dict(urllib.parse.parse_qsl(body))"` for application/x-www-form-urlencoded
+- **Nested extraction:** `json_path="data[0]"` for JSON with wrapper objects
+- **Languages:** .ts/.go/.rs/.java/.kt/.swift/.cs/.cpp/.dart/.rb/.json extensions set language
+- **Options:** Dict keys map to CLI flags: `{"readonly": True}` → `--readonly`, `{"nice_property_names": True}` → `--nice-property-names`. See `quicktype --help` for language-specific flags
+- **Common options:** TypeScript: `{"readonly": True, "prefer_types": True}`, Go: `{"omit_empty": True}`, Python: `{"pydantic_base_model": True}`
+- **Install:** `npm install -g quicktype` if command not found
+- **Pydantic models:** Use `to_model({id}, "models/model.py", "Model")` for Pydantic v2 instead
 
 ### inspect
 Inspect CDP events with full Python debugging.
@@ -123,25 +174,54 @@ events({"level": "error"})                # Console errors
 - **Decode data:** `inspect({id}, "base64.b64decode(data.get('params', {}).get('body', ''))")`
 
 ### js
-Execute JavaScript in the browser with optional promise handling.
+Execute JavaScript in the browser. Uses fresh scope by default to avoid redeclaration errors.
+
+#### Scope Behavior
+**Default (fresh scope)** - Each call runs in isolation:
+```python
+js("const x = 1")    # ✓ x isolated
+js("const x = 2")    # ✓ No error, fresh scope
+```
+
+**Persistent scope** - Variables survive across calls:
+```python
+js("var data = {count: 0}", persist=True)    # data persists
+js("data.count++", persist=True)              # Modifies data
+js("data.count", persist=True)                # Returns 1
+```
+
+**With browser element** - Always fresh scope:
+```python
+js("element.offsetWidth", selection=1)       # Use element #1
+js("element.classList", selection=2)         # Use element #2
+```
 
 #### Examples
 ```python
+# Basic queries
 js("document.title")                           # Get page title
-js("document.body.innerText.length")           # Get text length
 js("[...document.links].map(a => a.href)")    # Get all links
-js("fetch('/api').then(r => r.json())", await_promise=True)  # Async
+js("document.body.innerText.length")           # Text length
+
+# Async operations
+js("fetch('/api').then(r => r.json())", await_promise=True)
+
+# DOM manipulation (no return)
 js("document.querySelectorAll('.ad').forEach(e => e.remove())", wait_return=False)
-js("window.fetch = new Proxy(window.fetch, {get: (t, p) => console.log(p)})", wait_return=False)
+
+# Persistent state for multi-step operations
+js("var apiData = null", persist=True)
+js("fetch('/api').then(r => r.json()).then(d => apiData = d)", persist=True, await_promise=True)
+js("apiData.users.length", persist=True)
 ```
 
 #### Tips
-- **Extract all links:** `js("[...document.links].map(a => a.href)")`
-- **Get page text:** `js("document.body.innerText")`
-- **Find data attributes:** `js("[...document.querySelectorAll('[data-id]')].map(e => e.dataset)")`
-- **Monitor DOM:** `js("new MutationObserver(console.log).observe(document, {childList: true, subtree: true})", wait_return=False)`
-- **Hook fetch:** `js("window.fetch = new Proxy(fetch, {apply: (t, _, a) => {console.log(a); return t(...a)}})", wait_return=False)`
+- **Fresh scope:** Default behavior prevents const/let redeclaration errors
+- **Persistent state:** Use `persist=True` for multi-step operations or global hooks
+- **No return needed:** Set `wait_return=False` for DOM manipulation or hooks
+- **Browser selections:** Use `selection=N` with browser() to operate on selected elements
 - **Check console:** `console()` - see logged messages from JS execution
+- **Hook fetch:** `js("window.fetch = new Proxy(fetch, {apply: (t, _, a) => {console.log(a); return t(...a)}})", persist=True, wait_return=False)`
 
 ### fetch
 Control request interception for debugging and modification.
@@ -172,6 +252,29 @@ Show paused requests and responses.
 - **Modify request:** `resume({id}, modifications={'url': '...'})`
 - **Fail request:** `fail({id}, 'BlockedByClient')` - block the request
 
+### page
+Get current page information and navigate.
+
+#### Tips
+- **Navigate:** `navigate("https://example.com")` - go to URL
+- **Reload:** `reload()` or `reload(ignore_cache=True)` - refresh page
+- **History:** `back()`, `forward()`, `history()` - navigate history
+- **Execute JS:** `js("document.title")` - run JavaScript in page
+- **Monitor traffic:** `network()` - see requests, `console()` - see messages
+- **Switch page:** `pages()` then `connect(page=N)` - change to another tab
+- **Full status:** `status()` - connection details and event count
+
+### pages
+List available Chrome pages and manage connections.
+
+#### Tips
+- **Connect to page:** `connect(page={index})` - connect by index number
+- **Connect by ID:** `connect(page_id="{page_id}")` - stable across tab reordering
+- **Switch pages:** Just call `connect()` again - no need to disconnect first
+- **Check status:** `status()` - see current connection and event count
+- **Reconnect:** If connection lost, select page and `connect()` again
+- **Find page:** Look for title/URL in table - index stays consistent
+
 ### selections
 Browser element selections with prompt and analysis.
 

webtap/commands/_builders.py

@@ -22,7 +22,7 @@ Examples:
 
 Available builders:
   - table_response() - Tables with headers, warnings, tips
-  - info_response() - Key-value pairs with optional heading
+  - info_response() - Key-value pairs with optional heading and tips
   - error_response() - Errors with suggestions
   - success_response() - Success messages with details
   - warning_response() - Warnings with suggestions
@@ -83,6 +83,7 @@ def info_response(
     title: str | None = None,
     fields: dict | None = None,
     extra: str | None = None,
+    tips: list[str] | None = None,
 ) -> dict:
     """Build info display with key-value pairs.
 
@@ -90,6 +91,7 @@ def info_response(
         title: Optional info title
         fields: Dict of field names to values
         extra: Optional extra content (raw markdown)
+        tips: Optional developer tips/guidance
     """
     elements = []
 
@@ -107,6 +109,10 @@ def info_response(
     if not elements:
         elements.append({"type": "text", "content": "_No information available_"})
 
+    if tips:
+        elements.append({"type": "heading", "content": "Next Steps", "level": 3})
+        elements.append({"type": "list", "items": tips})
+
     return {"elements": elements}
 
 

webtap/commands/_code_generation.py

@@ -0,0 +1,110 @@
+"""Code generation utilities for transforming HTTP bodies into code.
+
+Pure transformation functions with no dependencies on services or state.
+Used by to_model(), quicktype(), and future code generation commands.
+"""
+
+import json
+from pathlib import Path
+from typing import Any
+
+
+def parse_json(content: str) -> tuple[Any, str | None]:
+    """Parse JSON string into Python object.
+
+    Args:
+        content: JSON string to parse.
+
+    Returns:
+        Tuple of (parsed_data, error_message).
+        On success: (data, None)
+        On failure: (None, error_string)
+
+    Examples:
+        data, error = parse_json('{"key": "value"}')
+        if error:
+            return error_response(error)
+    """
+    try:
+        return json.loads(content), None
+    except json.JSONDecodeError as e:
+        return None, f"Invalid JSON: {e}"
+
+
+def extract_json_path(data: Any, path: str) -> tuple[Any, str | None]:
+    """Extract nested data using simple bracket notation.
+
+    Supports paths like "data[0]", "results.users", or "data[0].items".
+
+    Args:
+        data: Dict or list to extract from.
+        path: Path using dot and bracket notation.
+
+    Returns:
+        Tuple of (extracted_data, error_message).
+        On success: (data, None)
+        On failure: (None, error_string)
+
+    Examples:
+        result, err = extract_json_path({"data": [1,2,3]}, "data[0]")
+        # result = 1, err = None
+
+        result, err = extract_json_path({"user": {"name": "Bob"}}, "user.name")
+        # result = "Bob", err = None
+    """
+    try:
+        parts = path.replace("[", ".").replace("]", "").split(".")
+        result = data
+        for part in parts:
+            if part:
+                if part.isdigit():
+                    result = result[int(part)]
+                else:
+                    result = result[part]
+        return result, None
+    except (KeyError, IndexError, TypeError) as e:
+        return None, f"JSON path '{path}' not found: {e}"
+
+
+def validate_generation_data(data: Any) -> tuple[bool, str | None]:
+    """Validate data structure for code generation.
+
+    Code generators (Pydantic, quicktype) require dict or list structures.
+
+    Args:
+        data: Data to validate.
+
+    Returns:
+        Tuple of (is_valid, error_message).
+        On success: (True, None)
+        On failure: (False, error_string)
+
+    Examples:
+        is_valid, error = validate_generation_data({"key": "value"})
+        # is_valid = True, error = None
+
+        is_valid, error = validate_generation_data("string")
+        # is_valid = False, error = "Data is str, not dict or list"
+    """
+    if not isinstance(data, (dict, list)):
+        return False, f"Data is {type(data).__name__}, not dict or list"
+    return True, None
+
+
+def ensure_output_directory(output: str) -> Path:
+    """Create output directory if needed, return resolved path.
+
+    Args:
+        output: Output file path (can be relative, use ~, etc.).
+
+    Returns:
+        Resolved absolute Path object.
+
+    Examples:
+        path = ensure_output_directory("~/models/user.py")
+        # Creates ~/models/ if it doesn't exist
+        # Returns Path("/home/user/models/user.py")
+    """
+    output_path = Path(output).expanduser().resolve()
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    return output_path

webtap/commands/body.py

@@ -11,11 +11,15 @@ mcp_desc = get_mcp_description("body")
 
 
 @app.command(display="markdown", fastmcp={"type": "tool", "description": mcp_desc} if mcp_desc else {"type": "tool"})
-def body(state, response: int, expr: str = None, decode: bool = True, cache: bool = True) -> dict:  # pyright: ignore[reportArgumentType]
-    """Fetch and analyze response body with Python expressions.
+def body(state, event: int, expr: str = None, decode: bool = True, cache: bool = True) -> dict:  # pyright: ignore[reportArgumentType]
+    """Fetch and analyze request or response body with Python expressions.
+
+    Automatically detects event type and fetches appropriate body:
+    - Request events (Network.requestWillBeSent): POST/PUT/PATCH request body
+    - Response events (Network.responseReceived): response body
 
     Args:
-        response: Response row ID from network() or requests()
+        event: Event row ID from network(), events(), or requests()
         expr: Optional Python expression with 'body' variable
         decode: Auto-decode base64 (default: True)
         cache: Use cached body (default: True)
@@ -28,7 +32,7 @@ def body(state, response: int, expr: str = None, decode: bool = True, cache: boo
 
     # Get body from service (with optional caching)
     body_service = state.service.body
-    result = body_service.get_response_body(response, use_cache=cache)
+    result = body_service.get_body(event, use_cache=cache)
 
     if "error" in result:
         return error_response(result["error"])
@@ -66,7 +70,7 @@ def body(state, response: int, expr: str = None, decode: bool = True, cache: boo
         # Build markdown response with body in code block
         # DATA-LEVEL TRUNCATION for memory/performance (as per refactor plan)
         MAX_BODY_SIZE = 5000  # Keep data-level truncation for large bodies
-        elements = [{"type": "heading", "content": "Response Body", "level": 2}]
+        elements = [{"type": "heading", "content": "Body", "level": 2}]
 
         # Try to detect content type and format appropriately
         content_preview = body_content[:100]

webtap/commands/connection.py

@@ -2,6 +2,7 @@
 
 from webtap.app import app
 from webtap.commands._builders import check_connection, info_response, table_response, error_response
+from webtap.commands._tips import get_tips
 
 
 @app.command(display="markdown", fastmcp={"type": "tool"})
@@ -132,12 +133,32 @@ def pages(state) -> dict:
         for i, p in enumerate(pages_list)
     ]
 
+    # Get contextual tips
+    tips = None
+    if rows:
+        # Find connected page or first page
+        connected_row = next((r for r in rows if r["Connected"] == "Yes"), rows[0])
+        page_index = connected_row["Index"]
+
+        # Get page_id for the example page
+        connected_page = next((p for p in pages_list if str(pages_list.index(p)) == page_index), None)
+        page_id = connected_page.get("id", "")[:6] if connected_page else ""
+
+        tips = get_tips("pages", context={"index": page_index, "page_id": page_id})
+
+    # Build contextual warnings
+    warnings = []
+    if any(r["Connected"] == "Yes" for r in rows):
+        warnings.append("Already connected - call connect(page=N) to switch pages")
+
     # Build markdown response
     return table_response(
         title="Chrome Pages",
         headers=["Index", "Title", "URL", "ID", "Connected"],
         rows=rows,
         summary=f"{len(pages_list)} page{'s' if len(pages_list) != 1 else ''} available",
+        warnings=warnings if warnings else None,
+        tips=tips,
     )
 
 

webtap/commands/javascript.py

@@ -23,39 +23,27 @@ def js(
     wait_return: bool = True,
     await_promise: bool = False,
 ) -> dict:
-    """Execute JavaScript in the browser with optional element selection.
+    """Execute JavaScript in the browser.
+
+    Uses fresh scope by default to avoid redeclaration errors. Set persist=True
+    to keep variables across calls. Use selection=N to operate on browser elements.
 
     Args:
-        code: JavaScript code to execute (use 'element' variable if selection provided)
-        selection: Browser element selection number (e.g., 1 for #1) - makes element available
-        persist: Keep variables in global scope across calls (default: False, uses fresh scope)
+        code: JavaScript code to execute
+        selection: Browser element selection number - makes 'element' variable available
+        persist: Keep variables in global scope across calls (default: False)
         wait_return: Wait for and return result (default: True)
         await_promise: Await promises before returning (default: False)
 
     Examples:
-        # Default: fresh scope (no redeclaration errors)
-        js("const x = 1")                              # ✓ x isolated
-        js("const x = 2")                              # ✓ No error, fresh scope
-        js("document.title")                           # Get page title
-        js("[...document.links].map(a => a.href)")    # Get all links
-
-        # With browser element selection (always fresh scope)
-        js("element.offsetWidth", selection=1)         # Use element #1
-        js("element.classList", selection=2)           # Use element #2
-        js("element.getBoundingClientRect()", selection=1)
-
-        # Persistent scope: keep variables across calls
-        js("var data = fetch('/api')", persist=True)   # data persists
-        js("data.json()", persist=True)                # Access data from previous call
-
-        # Async operations
-        js("fetch('/api').then(r => r.json())", await_promise=True)
-
-        # DOM manipulation (no return needed)
-        js("document.querySelectorAll('.ad').forEach(e => e.remove())", wait_return=False)
+        js("document.title")                                # Fresh scope (default)
+        js("var data = {count: 0}", persist=True)           # Persistent state
+        js("element.offsetWidth", selection=1)              # With browser element
+        js("fetch('/api')", await_promise=True)             # Async operation
+        js("element.remove()", selection=1, wait_return=False)  # No return needed
 
     Returns:
-        The evaluated result if wait_return=True, otherwise execution status
+        Evaluated result if wait_return=True, otherwise execution status
     """
     if error := check_connection(state):
         return error

webtap/commands/navigation.py

@@ -2,6 +2,7 @@
 
 from webtap.app import app
 from webtap.commands._builders import check_connection, info_response, table_response, error_response
+from webtap.commands._tips import get_tips
 
 
 @app.command(display="markdown", fastmcp={"type": "tool"})
@@ -146,6 +147,9 @@ def page(state) -> dict:
         except Exception:
             title = current.get("title", "")
 
+        # Get tips from TIPS.md
+        tips = get_tips("page")
+
         # Build formatted response
         return info_response(
             title=title or "Untitled Page",
@@ -154,6 +158,7 @@ def page(state) -> dict:
                 "ID": current.get("id", ""),
                 "Type": current.get("transitionType", ""),
             },
+            tips=tips,
         )
 
     return error_response("No navigation history available")