webtap-tool 0.7.1__py3-none-any.whl → 0.8.0__py3-none-any.whl

webtap/commands/_code_generation.py

@@ -0,0 +1,110 @@
+"""Code generation utilities for transforming HTTP bodies into code.
+
+Pure transformation functions with no dependencies on services or state.
+Used by to_model(), quicktype(), and future code generation commands.
+"""
+
+import json
+from pathlib import Path
+from typing import Any
+
+
+def parse_json(content: str) -> tuple[Any, str | None]:
+    """Parse JSON string into Python object.
+
+    Args:
+        content: JSON string to parse.
+
+    Returns:
+        Tuple of (parsed_data, error_message).
+        On success: (data, None)
+        On failure: (None, error_string)
+
+    Examples:
+        data, error = parse_json('{"key": "value"}')
+        if error:
+            return error_response(error)
+    """
+    try:
+        return json.loads(content), None
+    except json.JSONDecodeError as e:
+        return None, f"Invalid JSON: {e}"
+
+
+def extract_json_path(data: Any, path: str) -> tuple[Any, str | None]:
+    """Extract nested data using simple bracket notation.
+
+    Supports paths like "data[0]", "results.users", or "data[0].items".
+
+    Args:
+        data: Dict or list to extract from.
+        path: Path using dot and bracket notation.
+
+    Returns:
+        Tuple of (extracted_data, error_message).
+        On success: (data, None)
+        On failure: (None, error_string)
+
+    Examples:
+        result, err = extract_json_path({"data": [1,2,3]}, "data[0]")
+        # result = 1, err = None
+
+        result, err = extract_json_path({"user": {"name": "Bob"}}, "user.name")
+        # result = "Bob", err = None
+    """
+    try:
+        parts = path.replace("[", ".").replace("]", "").split(".")
+        result = data
+        for part in parts:
+            if part:
+                if part.isdigit():
+                    result = result[int(part)]
+                else:
+                    result = result[part]
+        return result, None
+    except (KeyError, IndexError, TypeError) as e:
+        return None, f"JSON path '{path}' not found: {e}"
+
+
+def validate_generation_data(data: Any) -> tuple[bool, str | None]:
+    """Validate data structure for code generation.
+
+    Code generators (Pydantic, quicktype) require dict or list structures.
+
+    Args:
+        data: Data to validate.
+
+    Returns:
+        Tuple of (is_valid, error_message).
+        On success: (True, None)
+        On failure: (False, error_string)
+
+    Examples:
+        is_valid, error = validate_generation_data({"key": "value"})
+        # is_valid = True, error = None
+
+        is_valid, error = validate_generation_data("string")
+        # is_valid = False, error = "Data is str, not dict or list"
+    """
+    if not isinstance(data, (dict, list)):
+        return False, f"Data is {type(data).__name__}, not dict or list"
+    return True, None
+
+
+def ensure_output_directory(output: str) -> Path:
+    """Create output directory if needed, return resolved path.
+
+    Args:
+        output: Output file path (can be relative, use ~, etc.).
+
+    Returns:
+        Resolved absolute Path object.
+
+    Examples:
+        path = ensure_output_directory("~/models/user.py")
+        # Creates ~/models/ if it doesn't exist
+        # Returns Path("/home/user/models/user.py")
+    """
+    output_path = Path(output).expanduser().resolve()
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    return output_path

webtap/commands/body.py

@@ -11,11 +11,15 @@ mcp_desc = get_mcp_description("body")
 
 
 @app.command(display="markdown", fastmcp={"type": "tool", "description": mcp_desc} if mcp_desc else {"type": "tool"})
-def body(state, response: int, expr: str = None, decode: bool = True, cache: bool = True) -> dict:  # pyright: ignore[reportArgumentType]
-    """Fetch and analyze response body with Python expressions.
+def body(state, event: int, expr: str = None, decode: bool = True, cache: bool = True) -> dict:  # pyright: ignore[reportArgumentType]
+    """Fetch and analyze request or response body with Python expressions.
+
+    Automatically detects event type and fetches appropriate body:
+    - Request events (Network.requestWillBeSent): POST/PUT/PATCH request body
+    - Response events (Network.responseReceived): response body
 
     Args:
-        response: Response row ID from network() or requests()
+        event: Event row ID from network(), events(), or requests()
         expr: Optional Python expression with 'body' variable
         decode: Auto-decode base64 (default: True)
         cache: Use cached body (default: True)
@@ -28,7 +32,7 @@ def body(state, response: int, expr: str = None, decode: bool = True, cache: boo
 
     # Get body from service (with optional caching)
     body_service = state.service.body
-    result = body_service.get_response_body(response, use_cache=cache)
+    result = body_service.get_body(event, use_cache=cache)
 
     if "error" in result:
         return error_response(result["error"])
@@ -66,7 +70,7 @@ def body(state, response: int, expr: str = None, decode: bool = True, cache: boo
         # Build markdown response with body in code block
         # DATA-LEVEL TRUNCATION for memory/performance (as per refactor plan)
         MAX_BODY_SIZE = 5000  # Keep data-level truncation for large bodies
-        elements = [{"type": "heading", "content": "Response Body", "level": 2}]
+        elements = [{"type": "heading", "content": "Body", "level": 2}]
 
         # Try to detect content type and format appropriately
         content_preview = body_content[:100]

webtap/commands/connection.py

@@ -2,6 +2,7 @@
 
 from webtap.app import app
 from webtap.commands._builders import check_connection, info_response, table_response, error_response
+from webtap.commands._tips import get_tips
 
 
 @app.command(display="markdown", fastmcp={"type": "tool"})
@@ -132,12 +133,32 @@ def pages(state) -> dict:
         for i, p in enumerate(pages_list)
     ]
 
+    # Get contextual tips
+    tips = None
+    if rows:
+        # Find connected page or first page
+        connected_row = next((r for r in rows if r["Connected"] == "Yes"), rows[0])
+        page_index = connected_row["Index"]
+
+        # Get page_id for the example page
+        connected_page = next((p for p in pages_list if str(pages_list.index(p)) == page_index), None)
+        page_id = connected_page.get("id", "")[:6] if connected_page else ""
+
+        tips = get_tips("pages", context={"index": page_index, "page_id": page_id})
+
+    # Build contextual warnings
+    warnings = []
+    if any(r["Connected"] == "Yes" for r in rows):
+        warnings.append("Already connected - call connect(page=N) to switch pages")
+
     # Build markdown response
     return table_response(
         title="Chrome Pages",
         headers=["Index", "Title", "URL", "ID", "Connected"],
         rows=rows,
         summary=f"{len(pages_list)} page{'s' if len(pages_list) != 1 else ''} available",
+        warnings=warnings if warnings else None,
+        tips=tips,
     )
 
 

webtap/commands/javascript.py

@@ -23,39 +23,27 @@ def js(
     wait_return: bool = True,
     await_promise: bool = False,
 ) -> dict:
-    """Execute JavaScript in the browser with optional element selection.
+    """Execute JavaScript in the browser.
+
+    Uses fresh scope by default to avoid redeclaration errors. Set persist=True
+    to keep variables across calls. Use selection=N to operate on browser elements.
 
     Args:
-        code: JavaScript code to execute (use 'element' variable if selection provided)
-        selection: Browser element selection number (e.g., 1 for #1) - makes element available
-        persist: Keep variables in global scope across calls (default: False, uses fresh scope)
+        code: JavaScript code to execute
+        selection: Browser element selection number - makes 'element' variable available
+        persist: Keep variables in global scope across calls (default: False)
         wait_return: Wait for and return result (default: True)
         await_promise: Await promises before returning (default: False)
 
     Examples:
-        # Default: fresh scope (no redeclaration errors)
-        js("const x = 1")                              # ✓ x isolated
-        js("const x = 2")                              # ✓ No error, fresh scope
-        js("document.title")                           # Get page title
-        js("[...document.links].map(a => a.href)")    # Get all links
-
-        # With browser element selection (always fresh scope)
-        js("element.offsetWidth", selection=1)         # Use element #1
-        js("element.classList", selection=2)           # Use element #2
-        js("element.getBoundingClientRect()", selection=1)
-
-        # Persistent scope: keep variables across calls
-        js("var data = fetch('/api')", persist=True)   # data persists
-        js("data.json()", persist=True)                # Access data from previous call
-
-        # Async operations
-        js("fetch('/api').then(r => r.json())", await_promise=True)
-
-        # DOM manipulation (no return needed)
-        js("document.querySelectorAll('.ad').forEach(e => e.remove())", wait_return=False)
+        js("document.title")                                # Fresh scope (default)
+        js("var data = {count: 0}", persist=True)           # Persistent state
+        js("element.offsetWidth", selection=1)              # With browser element
+        js("fetch('/api')", await_promise=True)             # Async operation
+        js("element.remove()", selection=1, wait_return=False)  # No return needed
 
     Returns:
-        The evaluated result if wait_return=True, otherwise execution status
+        Evaluated result if wait_return=True, otherwise execution status
     """
     if error := check_connection(state):
         return error

webtap/commands/navigation.py

@@ -2,6 +2,7 @@
 
 from webtap.app import app
 from webtap.commands._builders import check_connection, info_response, table_response, error_response
+from webtap.commands._tips import get_tips
 
 
 @app.command(display="markdown", fastmcp={"type": "tool"})
@@ -146,6 +147,9 @@ def page(state) -> dict:
         except Exception:
             title = current.get("title", "")
 
+        # Get tips from TIPS.md
+        tips = get_tips("page")
+
         # Build formatted response
         return info_response(
             title=title or "Untitled Page",
@@ -154,6 +158,7 @@ def page(state) -> dict:
                 "ID": current.get("id", ""),
                 "Type": current.get("transitionType", ""),
             },
+            tips=tips,
         )
 
     return error_response("No navigation history available")

webtap/commands/quicktype.py

@@ -0,0 +1,268 @@
+"""Generate types and schemas from HTTP response bodies using quicktype."""
+
+import json
+import shutil
+import subprocess
+from datetime import datetime
+from pathlib import Path
+from webtap.app import app
+from webtap.commands._builders import check_connection, success_response, error_response
+from webtap.commands._code_generation import ensure_output_directory
+from webtap.commands._tips import get_mcp_description
+
+
+mcp_desc = get_mcp_description("quicktype")
+
+# Header template for generated files
+HEADER_TEMPLATE = """Generated by WebTap from event {event_id}
+Source: {url}
+Method: {method}
+Generated: {timestamp}
+
+Do not edit manually."""
+
+# Comment syntax per language
+COMMENT_STYLES = {
+    "TypeScript": "//",
+    "Python": "#",
+    "Go": "//",
+    "Rust": "//",
+    "Java": "//",
+    "Kotlin": "//",
+    "Swift": "//",
+    "C#": "//",
+    "C++": "//",
+    "Dart": "//",
+    "Ruby": "#",
+    "JSON Schema": None,  # No comments in JSON
+}
+
+
+def _run_quicktype(
+    json_data: dict | list,
+    output: str,
+    type_name: str = None,  # pyright: ignore[reportArgumentType]
+    just_types: bool = True,
+    prefer_types: bool = True,
+    options: dict = None,  # pyright: ignore[reportArgumentType]
+) -> tuple[bool, str]:
+    """Run quicktype CLI with given parameters.
+
+    Args:
+        json_data: JSON data to convert
+        output: Output file path
+        type_name: Top-level type name
+        just_types: Generate only types, no serializers
+        prefer_types: Use 'type' instead of 'interface' for TypeScript
+        options: Additional quicktype flags
+
+    Returns:
+        Tuple of (success: bool, error_message: str)
+    """
+    # Check if quicktype is available
+    if not shutil.which("quicktype"):
+        return False, "quicktype CLI not found. Install with: npm install -g quicktype"
+
+    # Determine language from file extension
+    output_path = Path(output)
+    ext = output_path.suffix.lower()
+
+    # Build command
+    cmd = ["quicktype", "-o", str(output), "--src-lang", "json", "--top-level", type_name]
+
+    # Add opinionated defaults
+    if just_types:
+        cmd.append("--just-types")
+
+    # TypeScript-specific options
+    if ext in {".ts", ".tsx"} and prefer_types:
+        cmd.append("--prefer-types")
+
+    # Apply additional options
+    for key, val in (options or {}).items():
+        flag = f"--{key.replace('_', '-')}"
+        if val is True:
+            cmd.append(flag)
+        elif val is not False and val is not None:
+            cmd.extend([flag, str(val)])
+
+    # Run quicktype
+    try:
+        subprocess.run(
+            cmd,
+            input=json.dumps(json_data, indent=2),
+            capture_output=True,
+            text=True,
+            check=True,
+            timeout=30,
+        )
+        return True, ""
+    except subprocess.CalledProcessError as e:
+        return False, f"quicktype failed: {e.stderr.strip() if e.stderr else str(e)}"
+    except subprocess.TimeoutExpired:
+        return False, "quicktype timed out (>30s)"
+    except Exception as e:
+        return False, f"Unexpected error: {str(e)}"
+
+
+def _insert_header(state, event: int, output_path: Path, language: str) -> None:
+    """Insert language-aware header comment into generated file.
+
+    Args:
+        state: WebTap state with CDP session
+        event: Event row ID for metadata extraction
+        output_path: Path to generated file
+        language: Target language (e.g., "TypeScript", "Python")
+    """
+    if not HEADER_TEMPLATE or language not in COMMENT_STYLES:
+        return
+
+    comment_prefix = COMMENT_STYLES[language]
+    if not comment_prefix:  # Skip languages without comments (e.g., JSON)
+        return
+
+    try:
+        # Get event metadata
+        event_result = state.cdp.query("SELECT event FROM events WHERE rowid = ?", [event])
+        if not event_result:
+            return
+
+        event_data = json.loads(event_result[0][0])
+        params = event_data.get("params", {})
+
+        # Extract metadata from event
+        request_data = params.get("request", {})
+        response_data = params.get("response", {})
+
+        metadata = {
+            "event_id": event,
+            "url": request_data.get("url") or response_data.get("url", "N/A"),
+            "method": request_data.get("method", "N/A"),
+            "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+        }
+
+        # Format header text with metadata
+        header_text = HEADER_TEMPLATE.format(**metadata)
+
+        # Apply comment syntax to each line
+        header_lines = [
+            f"{comment_prefix} {line}" if line.strip() else comment_prefix for line in header_text.split("\n")
+        ]
+        header = "\n".join(header_lines)
+
+        # Prepend header to file
+        content = output_path.read_text()
+        output_path.write_text(header + "\n\n" + content)
+    except Exception:
+        # Silent failure - don't break generation if header fails
+        pass
+
+
+@app.command(display="markdown", fastmcp={"type": "tool", "description": mcp_desc} if mcp_desc else {"type": "tool"})
+def quicktype(
+    state,
+    event: int,
+    output: str,
+    type_name: str,
+    json_path: str = None,  # pyright: ignore[reportArgumentType]
+    expr: str = None,  # pyright: ignore[reportArgumentType]
+    just_types: bool = True,
+    prefer_types: bool = True,
+    options: dict = None,  # pyright: ignore[reportArgumentType]
+) -> dict:  # pyright: ignore[reportArgumentType]
+    """Generate types/schemas from request or response body using quicktype CLI.
+
+    Args:
+        event: Event row ID from network() or events()
+        output: Output file path (extension determines language: .ts, .py, .go, etc.)
+        type_name: Top-level type name (e.g., "User", "ApiResponse")
+        json_path: Optional JSON path to extract nested data (e.g., "data[0]")
+        expr: Optional Python expression to transform data (has 'body' and 'event' variables)
+        just_types: Generate only types, no serializers (default: True)
+        prefer_types: Use 'type' instead of 'interface' for TypeScript (default: True)
+        options: Additional quicktype flags as dict (e.g., {"readonly": True, "nice_property_names": True})
+
+    Examples:
+        quicktype(123, "types/User.ts", "User")                                    # TypeScript interface
+        quicktype(123, "models/customer.py", "Customer")                           # Python dataclass
+        quicktype(123, "api.go", "ApiResponse")                                    # Go struct
+        quicktype(123, "schema.json", "Schema")                                     # JSON Schema
+        quicktype(123, "types.ts", "User", json_path="data[0]")                    # Extract nested
+        quicktype(172, "types/Form.ts", "Form", expr="dict(urllib.parse.parse_qsl(body))")  # Parse form data
+        quicktype(123, "types.ts", "User", options={"readonly": True})             # Advanced options
+
+    Returns:
+        Success message with generation details
+    """
+    if error := check_connection(state):
+        return error
+
+    # Prepare data via service layer
+    result = state.service.body.prepare_for_generation(event, json_path, expr)
+    if result.get("error"):
+        return error_response(result["error"], suggestions=result.get("suggestions", []))
+
+    data = result["data"]
+
+    # Ensure output directory exists
+    output_path = ensure_output_directory(output)
+
+    # Run quicktype
+    success, error_msg = _run_quicktype(
+        json_data=data,
+        output=str(output_path),
+        type_name=type_name,
+        just_types=just_types,
+        prefer_types=prefer_types,
+        options=options,
+    )
+
+    if not success:
+        return error_response(
+            error_msg,
+            suggestions=[
+                "Check that quicktype is installed: npm install -g quicktype",
+                "Verify the JSON structure is valid",
+                "Try simplifying the data with json_path",
+            ],
+        )
+
+    # Detect language from extension
+    ext = output_path.suffix.lower()
+    lang_map = {
+        ".ts": "TypeScript",
+        ".tsx": "TypeScript",
+        ".py": "Python",
+        ".go": "Go",
+        ".rs": "Rust",
+        ".java": "Java",
+        ".kt": "Kotlin",
+        ".swift": "Swift",
+        ".cs": "C#",
+        ".cpp": "C++",
+        ".dart": "Dart",
+        ".rb": "Ruby",
+        ".json": "JSON Schema",
+    }
+    language = lang_map.get(ext, "Unknown")
+
+    # Add header comment with event metadata
+    _insert_header(state, event, output_path, language)
+
+    # Count lines in generated file
+    try:
+        file_content = output_path.read_text()
+        line_count = len(file_content.splitlines())
+    except Exception:
+        line_count = "unknown"
+
+    return success_response(
+        "Types generated successfully",
+        details={
+            "Output": str(output_path),
+            "Language": language,
+            "Type Name": type_name,
+            "Lines": line_count,
+            "Size": f"{output_path.stat().st_size} bytes",
+        },
+    )

webtap/commands/to_model.py

@@ -1,10 +1,10 @@
 """Generate Pydantic models from HTTP response bodies."""
 
 import json
-from pathlib import Path
 from datamodel_code_generator import generate, InputFileType, DataModelType
 from webtap.app import app
 from webtap.commands._builders import check_connection, success_response, error_response
+from webtap.commands._code_generation import ensure_output_directory
 from webtap.commands._tips import get_mcp_description
 
 
@@ -12,14 +12,20 @@ mcp_desc = get_mcp_description("to_model")
 
 
 @app.command(display="markdown", fastmcp={"type": "tool", "description": mcp_desc} if mcp_desc else {"type": "tool"})
-def to_model(state, response: int, output: str, model_name: str, json_path: str = None) -> dict:  # pyright: ignore[reportArgumentType]
-    """Generate Pydantic model from response body using datamodel-codegen.
+def to_model(state, event: int, output: str, model_name: str, json_path: str = None, expr: str = None) -> dict:  # pyright: ignore[reportArgumentType]
+    """Generate Pydantic model from request or response body using datamodel-codegen.
 
     Args:
-        response: Response row ID from network() table
+        event: Event row ID from network() or events()
         output: Output file path for generated model (e.g., "models/customers/group.py")
         model_name: Class name for generated model (e.g., "CustomerGroup")
-        json_path: Optional JSON path to extract nested data (e.g., "Data[0]")
+        json_path: Optional JSON path to extract nested data (e.g., "data[0]")
+        expr: Optional Python expression to transform data (has 'body' and 'event' variables)
+
+    Examples:
+        to_model(123, "models/user.py", "User", json_path="data[0]")
+        to_model(172, "models/form.py", "Form", expr="dict(urllib.parse.parse_qsl(body))")
+        to_model(123, "models/clean.py", "Clean", expr="{k: v for k, v in json.loads(body).items() if k != 'meta'}")
 
     Returns:
         Success message with generation details
@@ -27,73 +33,15 @@ def to_model(state, response: int, output: str, model_name: str, json_path: str
     if error := check_connection(state):
         return error
 
-    # Get response body from service
-    body_service = state.service.body
-    result = body_service.get_response_body(response, use_cache=True)
-
-    if "error" in result:
-        return error_response(result["error"])
-
-    body_content = result.get("body", "")
-    is_base64 = result.get("base64Encoded", False)
-
-    # Decode if needed
-    if is_base64:
-        decoded = body_service.decode_body(body_content, is_base64)
-        if isinstance(decoded, bytes):
-            return error_response(
-                "Response body is binary",
-                suggestions=["Only JSON responses can be converted to models", "Try a different response"],
-            )
-        body_content = decoded
+    # Prepare data via service layer
+    result = state.service.body.prepare_for_generation(event, json_path, expr)
+    if result.get("error"):
+        return error_response(result["error"], suggestions=result.get("suggestions", []))
 
-    # Parse JSON
-    try:
-        data = json.loads(body_content)
-    except json.JSONDecodeError as e:
-        return error_response(
-            f"Invalid JSON: {e}",
-            suggestions=[
-                "Response must be valid JSON",
-                "Check the response with body() first",
-                "Try a different response",
-            ],
-        )
-
-    # Extract JSON path if specified
-    if json_path:
-        try:
-            # Support simple bracket notation like "Data[0]"
-            parts = json_path.replace("[", ".").replace("]", "").split(".")
-            for part in parts:
-                if part:
-                    if part.isdigit():
-                        data = data[int(part)]
-                    else:
-                        data = data[part]
-        except (KeyError, IndexError, TypeError) as e:
-            return error_response(
-                f"JSON path extraction failed: {e}",
-                suggestions=[
-                    f"Path '{json_path}' not found in response",
-                    "Check the response structure with body()",
-                    'Try a simpler path like "Data" or "Data[0]"',
-                ],
-            )
-
-    # Ensure data is dict or list for model generation
-    if not isinstance(data, (dict, list)):
-        return error_response(
-            f"Extracted data is {type(data).__name__}, not dict or list",
-            suggestions=[
-                "Model generation requires dict or list structure",
-                "Adjust json_path to extract a complex object",
-            ],
-        )
+    data = result["data"]
 
-    # Create output directory if needed
-    output_path = Path(output).expanduser().resolve()
-    output_path.parent.mkdir(parents=True, exist_ok=True)
+    # Ensure output directory exists
+    output_path = ensure_output_directory(output)
 
     # Generate model using datamodel-codegen Python API
     try:
@@ -103,10 +51,10 @@ def to_model(state, response: int, output: str, model_name: str, json_path: str
             input_filename="response.json",
             output=output_path,
             output_model_type=DataModelType.PydanticV2BaseModel,
-            class_name=model_name,  # Set generated class name
-            snake_case_field=True,  # Convert to snake_case
-            use_standard_collections=True,  # Use list instead of List
-            use_union_operator=True,  # Use | instead of Union
+            class_name=model_name,
+            snake_case_field=True,
+            use_standard_collections=True,
+            use_union_operator=True,
         )
     except Exception as e:
         return error_response(
@@ -121,7 +69,7 @@ def to_model(state, response: int, output: str, model_name: str, json_path: str
     # Count fields in generated model
     try:
         model_content = output_path.read_text()
-        field_count = model_content.count(": ")  # Count field definitions
+        field_count = model_content.count(": ")
     except Exception:
         field_count = "unknown"