webtap-tool 0.7.0__py3-none-any.whl → 0.8.0__py3-none-any.whl

webtap/commands/quicktype.py

@@ -0,0 +1,268 @@
+"""Generate types and schemas from HTTP response bodies using quicktype."""
+
+import json
+import shutil
+import subprocess
+from datetime import datetime
+from pathlib import Path
+from webtap.app import app
+from webtap.commands._builders import check_connection, success_response, error_response
+from webtap.commands._code_generation import ensure_output_directory
+from webtap.commands._tips import get_mcp_description
+
+
+mcp_desc = get_mcp_description("quicktype")
+
+# Header template for generated files
+HEADER_TEMPLATE = """Generated by WebTap from event {event_id}
+Source: {url}
+Method: {method}
+Generated: {timestamp}
+
+Do not edit manually."""
+
+# Comment syntax per language
+COMMENT_STYLES = {
+    "TypeScript": "//",
+    "Python": "#",
+    "Go": "//",
+    "Rust": "//",
+    "Java": "//",
+    "Kotlin": "//",
+    "Swift": "//",
+    "C#": "//",
+    "C++": "//",
+    "Dart": "//",
+    "Ruby": "#",
+    "JSON Schema": None,  # No comments in JSON
+}
+
+
+def _run_quicktype(
+    json_data: dict | list,
+    output: str,
+    type_name: str = None,  # pyright: ignore[reportArgumentType]
+    just_types: bool = True,
+    prefer_types: bool = True,
+    options: dict = None,  # pyright: ignore[reportArgumentType]
+) -> tuple[bool, str]:
+    """Run quicktype CLI with given parameters.
+
+    Args:
+        json_data: JSON data to convert
+        output: Output file path
+        type_name: Top-level type name
+        just_types: Generate only types, no serializers
+        prefer_types: Use 'type' instead of 'interface' for TypeScript
+        options: Additional quicktype flags
+
+    Returns:
+        Tuple of (success: bool, error_message: str)
+    """
+    # Check if quicktype is available
+    if not shutil.which("quicktype"):
+        return False, "quicktype CLI not found. Install with: npm install -g quicktype"
+
+    # Determine language from file extension
+    output_path = Path(output)
+    ext = output_path.suffix.lower()
+
+    # Build command
+    cmd = ["quicktype", "-o", str(output), "--src-lang", "json", "--top-level", type_name]
+
+    # Add opinionated defaults
+    if just_types:
+        cmd.append("--just-types")
+
+    # TypeScript-specific options
+    if ext in {".ts", ".tsx"} and prefer_types:
+        cmd.append("--prefer-types")
+
+    # Apply additional options
+    for key, val in (options or {}).items():
+        flag = f"--{key.replace('_', '-')}"
+        if val is True:
+            cmd.append(flag)
+        elif val is not False and val is not None:
+            cmd.extend([flag, str(val)])
+
+    # Run quicktype
+    try:
+        subprocess.run(
+            cmd,
+            input=json.dumps(json_data, indent=2),
+            capture_output=True,
+            text=True,
+            check=True,
+            timeout=30,
+        )
+        return True, ""
+    except subprocess.CalledProcessError as e:
+        return False, f"quicktype failed: {e.stderr.strip() if e.stderr else str(e)}"
+    except subprocess.TimeoutExpired:
+        return False, "quicktype timed out (>30s)"
+    except Exception as e:
+        return False, f"Unexpected error: {str(e)}"
+
+
+def _insert_header(state, event: int, output_path: Path, language: str) -> None:
+    """Insert language-aware header comment into generated file.
+
+    Args:
+        state: WebTap state with CDP session
+        event: Event row ID for metadata extraction
+        output_path: Path to generated file
+        language: Target language (e.g., "TypeScript", "Python")
+    """
+    if not HEADER_TEMPLATE or language not in COMMENT_STYLES:
+        return
+
+    comment_prefix = COMMENT_STYLES[language]
+    if not comment_prefix:  # Skip languages without comments (e.g., JSON)
+        return
+
+    try:
+        # Get event metadata
+        event_result = state.cdp.query("SELECT event FROM events WHERE rowid = ?", [event])
+        if not event_result:
+            return
+
+        event_data = json.loads(event_result[0][0])
+        params = event_data.get("params", {})
+
+        # Extract metadata from event
+        request_data = params.get("request", {})
+        response_data = params.get("response", {})
+
+        metadata = {
+            "event_id": event,
+            "url": request_data.get("url") or response_data.get("url", "N/A"),
+            "method": request_data.get("method", "N/A"),
+            "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+        }
+
+        # Format header text with metadata
+        header_text = HEADER_TEMPLATE.format(**metadata)
+
+        # Apply comment syntax to each line
+        header_lines = [
+            f"{comment_prefix} {line}" if line.strip() else comment_prefix for line in header_text.split("\n")
+        ]
+        header = "\n".join(header_lines)
+
+        # Prepend header to file
+        content = output_path.read_text()
+        output_path.write_text(header + "\n\n" + content)
+    except Exception:
+        # Silent failure - don't break generation if header fails
+        pass
+
+
+@app.command(display="markdown", fastmcp={"type": "tool", "description": mcp_desc} if mcp_desc else {"type": "tool"})
+def quicktype(
+    state,
+    event: int,
+    output: str,
+    type_name: str,
+    json_path: str = None,  # pyright: ignore[reportArgumentType]
+    expr: str = None,  # pyright: ignore[reportArgumentType]
+    just_types: bool = True,
+    prefer_types: bool = True,
+    options: dict = None,  # pyright: ignore[reportArgumentType]
+) -> dict:  # pyright: ignore[reportArgumentType]
+    """Generate types/schemas from request or response body using quicktype CLI.
+
+    Args:
+        event: Event row ID from network() or events()
+        output: Output file path (extension determines language: .ts, .py, .go, etc.)
+        type_name: Top-level type name (e.g., "User", "ApiResponse")
+        json_path: Optional JSON path to extract nested data (e.g., "data[0]")
+        expr: Optional Python expression to transform data (has 'body' and 'event' variables)
+        just_types: Generate only types, no serializers (default: True)
+        prefer_types: Use 'type' instead of 'interface' for TypeScript (default: True)
+        options: Additional quicktype flags as dict (e.g., {"readonly": True, "nice_property_names": True})
+
+    Examples:
+        quicktype(123, "types/User.ts", "User")                                    # TypeScript interface
+        quicktype(123, "models/customer.py", "Customer")                           # Python dataclass
+        quicktype(123, "api.go", "ApiResponse")                                    # Go struct
+        quicktype(123, "schema.json", "Schema")                                     # JSON Schema
+        quicktype(123, "types.ts", "User", json_path="data[0]")                    # Extract nested
+        quicktype(172, "types/Form.ts", "Form", expr="dict(urllib.parse.parse_qsl(body))")  # Parse form data
+        quicktype(123, "types.ts", "User", options={"readonly": True})             # Advanced options
+
+    Returns:
+        Success message with generation details
+    """
+    if error := check_connection(state):
+        return error
+
+    # Prepare data via service layer
+    result = state.service.body.prepare_for_generation(event, json_path, expr)
+    if result.get("error"):
+        return error_response(result["error"], suggestions=result.get("suggestions", []))
+
+    data = result["data"]
+
+    # Ensure output directory exists
+    output_path = ensure_output_directory(output)
+
+    # Run quicktype
+    success, error_msg = _run_quicktype(
+        json_data=data,
+        output=str(output_path),
+        type_name=type_name,
+        just_types=just_types,
+        prefer_types=prefer_types,
+        options=options,
+    )
+
+    if not success:
+        return error_response(
+            error_msg,
+            suggestions=[
+                "Check that quicktype is installed: npm install -g quicktype",
+                "Verify the JSON structure is valid",
+                "Try simplifying the data with json_path",
+            ],
+        )
+
+    # Detect language from extension
+    ext = output_path.suffix.lower()
+    lang_map = {
+        ".ts": "TypeScript",
+        ".tsx": "TypeScript",
+        ".py": "Python",
+        ".go": "Go",
+        ".rs": "Rust",
+        ".java": "Java",
+        ".kt": "Kotlin",
+        ".swift": "Swift",
+        ".cs": "C#",
+        ".cpp": "C++",
+        ".dart": "Dart",
+        ".rb": "Ruby",
+        ".json": "JSON Schema",
+    }
+    language = lang_map.get(ext, "Unknown")
+
+    # Add header comment with event metadata
+    _insert_header(state, event, output_path, language)
+
+    # Count lines in generated file
+    try:
+        file_content = output_path.read_text()
+        line_count = len(file_content.splitlines())
+    except Exception:
+        line_count = "unknown"
+
+    return success_response(
+        "Types generated successfully",
+        details={
+            "Output": str(output_path),
+            "Language": language,
+            "Type Name": type_name,
+            "Lines": line_count,
+            "Size": f"{output_path.stat().st_size} bytes",
+        },
+    )

webtap/commands/to_model.py

@@ -1,10 +1,10 @@
 """Generate Pydantic models from HTTP response bodies."""
 
 import json
-from pathlib import Path
 from datamodel_code_generator import generate, InputFileType, DataModelType
 from webtap.app import app
 from webtap.commands._builders import check_connection, success_response, error_response
+from webtap.commands._code_generation import ensure_output_directory
 from webtap.commands._tips import get_mcp_description
 
 
@@ -12,14 +12,20 @@ mcp_desc = get_mcp_description("to_model")
 
 
 @app.command(display="markdown", fastmcp={"type": "tool", "description": mcp_desc} if mcp_desc else {"type": "tool"})
-def to_model(state, response: int, output: str, model_name: str, json_path: str = None) -> dict:  # pyright: ignore[reportArgumentType]
-    """Generate Pydantic model from response body using datamodel-codegen.
+def to_model(state, event: int, output: str, model_name: str, json_path: str = None, expr: str = None) -> dict:  # pyright: ignore[reportArgumentType]
+    """Generate Pydantic model from request or response body using datamodel-codegen.
 
     Args:
-        response: Response row ID from network() table
+        event: Event row ID from network() or events()
         output: Output file path for generated model (e.g., "models/customers/group.py")
         model_name: Class name for generated model (e.g., "CustomerGroup")
-        json_path: Optional JSON path to extract nested data (e.g., "Data[0]")
+        json_path: Optional JSON path to extract nested data (e.g., "data[0]")
+        expr: Optional Python expression to transform data (has 'body' and 'event' variables)
+
+    Examples:
+        to_model(123, "models/user.py", "User", json_path="data[0]")
+        to_model(172, "models/form.py", "Form", expr="dict(urllib.parse.parse_qsl(body))")
+        to_model(123, "models/clean.py", "Clean", expr="{k: v for k, v in json.loads(body).items() if k != 'meta'}")
 
     Returns:
         Success message with generation details
@@ -27,73 +33,15 @@ def to_model(state, response: int, output: str, model_name: str, json_path: str
     if error := check_connection(state):
         return error
 
-    # Get response body from service
-    body_service = state.service.body
-    result = body_service.get_response_body(response, use_cache=True)
-
-    if "error" in result:
-        return error_response(result["error"])
-
-    body_content = result.get("body", "")
-    is_base64 = result.get("base64Encoded", False)
-
-    # Decode if needed
-    if is_base64:
-        decoded = body_service.decode_body(body_content, is_base64)
-        if isinstance(decoded, bytes):
-            return error_response(
-                "Response body is binary",
-                suggestions=["Only JSON responses can be converted to models", "Try a different response"],
-            )
-        body_content = decoded
+    # Prepare data via service layer
+    result = state.service.body.prepare_for_generation(event, json_path, expr)
+    if result.get("error"):
+        return error_response(result["error"], suggestions=result.get("suggestions", []))
 
-    # Parse JSON
-    try:
-        data = json.loads(body_content)
-    except json.JSONDecodeError as e:
-        return error_response(
-            f"Invalid JSON: {e}",
-            suggestions=[
-                "Response must be valid JSON",
-                "Check the response with body() first",
-                "Try a different response",
-            ],
-        )
-
-    # Extract JSON path if specified
-    if json_path:
-        try:
-            # Support simple bracket notation like "Data[0]"
-            parts = json_path.replace("[", ".").replace("]", "").split(".")
-            for part in parts:
-                if part:
-                    if part.isdigit():
-                        data = data[int(part)]
-                    else:
-                        data = data[part]
-        except (KeyError, IndexError, TypeError) as e:
-            return error_response(
-                f"JSON path extraction failed: {e}",
-                suggestions=[
-                    f"Path '{json_path}' not found in response",
-                    "Check the response structure with body()",
-                    'Try a simpler path like "Data" or "Data[0]"',
-                ],
-            )
-
-    # Ensure data is dict or list for model generation
-    if not isinstance(data, (dict, list)):
-        return error_response(
-            f"Extracted data is {type(data).__name__}, not dict or list",
-            suggestions=[
-                "Model generation requires dict or list structure",
-                "Adjust json_path to extract a complex object",
-            ],
-        )
+    data = result["data"]
 
-    # Create output directory if needed
-    output_path = Path(output).expanduser().resolve()
-    output_path.parent.mkdir(parents=True, exist_ok=True)
+    # Ensure output directory exists
+    output_path = ensure_output_directory(output)
 
     # Generate model using datamodel-codegen Python API
     try:
@@ -103,10 +51,10 @@ def to_model(state, response: int, output: str, model_name: str, json_path: str
             input_filename="response.json",
             output=output_path,
             output_model_type=DataModelType.PydanticV2BaseModel,
-            class_name=model_name,  # Set generated class name
-            snake_case_field=True,  # Convert to snake_case
-            use_standard_collections=True,  # Use list instead of List
-            use_union_operator=True,  # Use | instead of Union
+            class_name=model_name,
+            snake_case_field=True,
+            use_standard_collections=True,
+            use_union_operator=True,
         )
     except Exception as e:
         return error_response(
@@ -121,7 +69,7 @@ def to_model(state, response: int, output: str, model_name: str, json_path: str
     # Count fields in generated model
     try:
         model_content = output_path.read_text()
-        field_count = model_content.count(": ")  # Count field definitions
+        field_count = model_content.count(": ")
     except Exception:
         field_count = "unknown"
 

webtap/services/body.py

@@ -19,17 +19,25 @@ class BodyService:
         self.cdp: CDPSession | None = None
         self._body_cache: dict[str, dict] = {}
 
-    def get_response_body(self, rowid: int, use_cache: bool = True) -> dict:
-        """Fetch response body for a response.
+    def get_body(self, rowid: int, use_cache: bool = True) -> dict:
+        """Fetch request or response body for an event.
+
+        Automatically detects event type and fetches appropriate body:
+        - Network.requestWillBeSent: request body (POST data)
+        - Network.responseReceived: response body
+        - Fetch.requestPaused: request or response body based on stage
 
         Args:
-            rowid: Row ID from events table (Network or Fetch response)
+            rowid: Row ID from events table
             use_cache: Whether to use cached body if available
+
+        Returns:
+            Dict with 'body' (str), 'base64Encoded' (bool), and 'event' (dict), or 'error' (str)
         """
         if not self.cdp:
             return {"error": "No CDP session"}
 
-        # Get event from DB to extract requestId
+        # Get event from DB to extract requestId and method
         result = self.cdp.query("SELECT event FROM events WHERE rowid = ?", [rowid])
 
         if not result:
@@ -42,45 +50,89 @@ class BodyService:
 
         method = event_data.get("method", "")
         params = event_data.get("params", {})
+        request_id = params.get("requestId")
+
+        if not request_id:
+            return {"error": "No requestId in event"}
+
+        # Check cache first (cache includes event_data)
+        cache_key = f"{request_id}:{method}"
+        if use_cache and cache_key in self._body_cache:
+            logger.debug(f"Using cached body for {cache_key}")
+            return self._body_cache[cache_key]
+
+        # Handle request body (POST data)
+        if method == "Network.requestWillBeSent":
+            request = params.get("request", {})
+
+            # Check inline postData first (may be present for small bodies)
+            if request.get("postData"):
+                logger.debug(f"Using inline postData for {request_id}")
+                body_data = {"body": request["postData"], "base64Encoded": False, "event": event_data}
+                if use_cache:
+                    self._body_cache[cache_key] = body_data
+                return body_data
+
+            # Check if request has POST data
+            if not request.get("hasPostData"):
+                return {"error": "No POST data in this request (GET or no body)"}
+
+            # Try to fetch POST data via CDP
+            try:
+                logger.debug(f"Fetching POST data for {request_id} using Network.getRequestPostData")
+                result = self.cdp.execute("Network.getRequestPostData", {"requestId": request_id})
+                body_data = {"body": result.get("postData", ""), "base64Encoded": False, "event": event_data}
+
+                if use_cache:
+                    self._body_cache[cache_key] = body_data
+                    logger.debug(f"Cached POST data for {request_id}")
+
+                return body_data
 
-        # Handle both Fetch and Network events
-        if method == "Fetch.requestPaused":
+            except Exception as e:
+                error_msg = str(e)
+                logger.error(f"Failed to fetch POST data for {request_id}: {error_msg}")
+                # Provide helpful error message
+                if "No resource with given identifier found" in error_msg:
+                    return {"error": "POST data not available (possibly too large or not captured by CDP)"}
+                return {"error": f"Failed to fetch POST data: {error_msg}"}
+
+        # Handle response body
+        elif method == "Fetch.requestPaused":
             # Fetch interception - verify it's response stage
             if "responseStatusCode" not in params:
                 return {"error": "Not a response stage event (no responseStatusCode)"}
-            request_id = params.get("requestId")
             domain = "Fetch"
+
         elif method == "Network.responseReceived":
             # Regular network response
-            request_id = params.get("requestId")
             domain = "Network"
-        else:
-            return {"error": f"Not a response event (method: {method})"}
 
-        if not request_id:
-            return {"error": "No requestId in event"}
-
-        # Check cache
-        if use_cache and request_id in self._body_cache:
-            logger.debug(f"Using cached body for {request_id}")
-            return self._body_cache[request_id]
+        else:
+            # Unknown event type - return empty body but include event for expr access
+            logger.debug(f"Unknown event type {method} - returning empty body with event data")
+            return {"body": "", "base64Encoded": False, "event": event_data}
 
+        # Fetch response body from CDP
         try:
-            # Fetch body from CDP using appropriate domain
-            logger.debug(f"Fetching body for {request_id} using {domain}.getResponseBody")
+            logger.debug(f"Fetching response body for {request_id} using {domain}.getResponseBody")
             result = self.cdp.execute(f"{domain}.getResponseBody", {"requestId": request_id})
 
-            body_data = {"body": result.get("body", ""), "base64Encoded": result.get("base64Encoded", False)}
+            body_data = {
+                "body": result.get("body", ""),
+                "base64Encoded": result.get("base64Encoded", False),
+                "event": event_data,
+            }
 
-            # Cache it for this request
+            # Cache it
             if use_cache:
-                self._body_cache[request_id] = body_data
-                logger.debug(f"Cached body for {request_id}")
+                self._body_cache[cache_key] = body_data
+                logger.debug(f"Cached response body for {request_id}")
 
             return body_data
 
         except Exception as e:
-            logger.error(f"Failed to fetch body for {request_id}: {e}")
+            logger.error(f"Failed to fetch response body for {request_id}: {e}")
             return {"error": str(e)}
 
     def clear_cache(self):
@@ -111,3 +163,136 @@ class BodyService:
         except Exception as e:
             logger.error(f"Failed to decode base64 body: {e}")
             return body_content  # Return original if decode fails
+
+    def prepare_for_generation(
+        self,
+        event: int,
+        json_path: str = None,  # pyright: ignore[reportArgumentType]
+        expr: str = None,  # pyright: ignore[reportArgumentType]
+    ) -> dict:
+        """Prepare HTTP body for code generation.
+
+        Orchestrates the complete pipeline:
+        1. Fetch body + event from CDP
+        2. Decode base64 if needed
+        3. Transform via expression OR validate and parse JSON
+        4. Extract nested data via json_path
+        5. Validate data structure (dict/list)
+
+        Args:
+            event: Event row ID from network() or events()
+            json_path: Optional JSON path for nested extraction (e.g., "data[0]")
+            expr: Optional Python expression with 'body' and 'event' variables
+
+        Returns:
+            Dict with 'data' (dict|list) on success, or 'error' (str) on failure.
+            May include 'suggestions' (list[str]) for error guidance.
+
+        Examples:
+            result = body_service.prepare_for_generation(123, json_path="data[0]")
+            if result.get("error"):
+                return error_response(result["error"], suggestions=result.get("suggestions"))
+            data = result["data"]
+        """
+        # 1. Fetch body + event from CDP
+        result = self.get_body(event, use_cache=True)
+        if "error" in result:
+            return {"error": result["error"], "suggestions": [], "data": None}
+
+        body_content = result["body"]
+        is_base64 = result["base64Encoded"]
+        event_data = result["event"]
+
+        # 2. Decode if base64
+        if is_base64:
+            decoded = self.decode_body(body_content, is_base64)
+            if isinstance(decoded, bytes):
+                return {
+                    "error": "Body is binary content",
+                    "suggestions": [
+                        "Only text/JSON can be converted to code",
+                        "Try a different event with text content",
+                    ],
+                    "data": None,
+                }
+            body_content = decoded
+
+        # 3. Transform via expression OR validate and parse JSON
+        if expr:
+            # Use expression evaluation from _utils
+            from webtap.commands._utils import evaluate_expression
+
+            try:
+                namespace = {"body": body_content, "event": event_data}
+                data, _ = evaluate_expression(expr, namespace)
+            except Exception as e:
+                return {
+                    "error": f"Expression evaluation failed: {e}",
+                    "suggestions": [
+                        "Check your expression syntax",
+                        "Variables available: 'body' (str), 'event' (dict)",
+                        "Example: dict(urllib.parse.parse_qsl(body))",
+                        "Example: json.loads(body)['data'][0]",
+                    ],
+                    "data": None,
+                }
+        else:
+            # Validate body is not empty before parsing
+            if not body_content.strip():
+                return {
+                    "error": "Body is empty",
+                    "suggestions": [
+                        "Use expr to extract data from event for non-HTTP events",
+                        "Example: expr=\"json.loads(event['params']['response']['payloadData'])\"",
+                        "Check the event structure with inspect() first",
+                    ],
+                    "data": None,
+                }
+
+            # Parse as JSON
+            from webtap.commands._code_generation import parse_json
+
+            data, error = parse_json(body_content)
+            if error:
+                return {
+                    "error": error,
+                    "suggestions": [
+                        "Body must be valid JSON or use expr to transform it",
+                        'For form data: expr="dict(urllib.parse.parse_qsl(body))"',
+                        "Check the body with body() command first",
+                    ],
+                    "data": None,
+                }
+
+        # 4. Extract nested path if specified
+        if json_path:
+            from webtap.commands._code_generation import extract_json_path
+
+            data, error = extract_json_path(data, json_path)
+            if error:
+                return {
+                    "error": error,
+                    "suggestions": [
+                        f"Path '{json_path}' not found in body",
+                        "Check the body structure with body() command",
+                        'Try a simpler path like "data" or "data[0]"',
+                    ],
+                    "data": None,
+                }
+
+        # 5. Validate structure
+        from webtap.commands._code_generation import validate_generation_data
+
+        is_valid, error = validate_generation_data(data)
+        if not is_valid:
+            return {
+                "error": error,
+                "suggestions": [
+                    "Code generation requires dict or list structure",
+                    "Adjust json_path to extract a complex object",
+                    "Or use expr to transform data into dict/list",
+                ],
+                "data": None,
+            }
+
+        return {"data": data, "error": None, "suggestions": []}