webtap-tool 0.11.0__py3-none-any.whl

webtap/commands/_code_generation.py

@@ -0,0 +1,110 @@
+"""Code generation utilities for transforming HTTP bodies into code.
+
+Pure transformation functions with no dependencies on services or state.
+Used by to_model(), quicktype(), and future code generation commands.
+"""
+
+import json
+from pathlib import Path
+from typing import Any
+
+
+def parse_json(content: str) -> tuple[Any, str | None]:
+    """Parse JSON string into Python object.
+
+    Args:
+        content: JSON string to parse.
+
+    Returns:
+        Tuple of (parsed_data, error_message).
+        On success: (data, None)
+        On failure: (None, error_string)
+
+    Examples:
+        data, error = parse_json('{"key": "value"}')
+        if error:
+            return error_response(error)
+    """
+    try:
+        return json.loads(content), None
+    except json.JSONDecodeError as e:
+        return None, f"Invalid JSON: {e}"
+
+
+def extract_json_path(data: Any, path: str) -> tuple[Any, str | None]:
+    """Extract nested data using simple bracket notation.
+
+    Supports paths like "data[0]", "results.users", or "data[0].items".
+
+    Args:
+        data: Dict or list to extract from.
+        path: Path using dot and bracket notation.
+
+    Returns:
+        Tuple of (extracted_data, error_message).
+        On success: (data, None)
+        On failure: (None, error_string)
+
+    Examples:
+        result, err = extract_json_path({"data": [1,2,3]}, "data[0]")
+        # result = 1, err = None
+
+        result, err = extract_json_path({"user": {"name": "Bob"}}, "user.name")
+        # result = "Bob", err = None
+    """
+    try:
+        parts = path.replace("[", ".").replace("]", "").split(".")
+        result = data
+        for part in parts:
+            if part:
+                if part.isdigit():
+                    result = result[int(part)]
+                else:
+                    result = result[part]
+        return result, None
+    except (KeyError, IndexError, TypeError) as e:
+        return None, f"JSON path '{path}' not found: {e}"
+
+
+def validate_generation_data(data: Any) -> tuple[bool, str | None]:
+    """Validate data structure for code generation.
+
+    Code generators (Pydantic, quicktype) require dict or list structures.
+
+    Args:
+        data: Data to validate.
+
+    Returns:
+        Tuple of (is_valid, error_message).
+        On success: (True, None)
+        On failure: (False, error_string)
+
+    Examples:
+        is_valid, error = validate_generation_data({"key": "value"})
+        # is_valid = True, error = None
+
+        is_valid, error = validate_generation_data("string")
+        # is_valid = False, error = "Data is str, not dict or list"
+    """
+    if not isinstance(data, (dict, list)):
+        return False, f"Data is {type(data).__name__}, not dict or list"
+    return True, None
+
+
+def ensure_output_directory(output: str) -> Path:
+    """Create output directory if needed, return resolved path.
+
+    Args:
+        output: Output file path (can be relative, use ~, etc.).
+
+    Returns:
+        Resolved absolute Path object.
+
+    Examples:
+        path = ensure_output_directory("~/models/user.py")
+        # Creates ~/models/ if it doesn't exist
+        # Returns Path("/home/user/models/user.py")
+    """
+    output_path = Path(output).expanduser().resolve()
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    return output_path

webtap/commands/_tips.py

@@ -0,0 +1,147 @@
+"""Parser for TIPS.md documentation.
+
+This module reads TIPS.md and provides:
+- MCP descriptions for commands
+- Developer tips for command responses
+- Pre-imported libraries documentation
+"""
+
+import re
+from pathlib import Path
+from typing import Dict, List, Optional
+
+
+class TipsParser:
+    """Parse TIPS.md for command documentation."""
+
+    def __init__(self):
+        # TIPS.md is in the same directory as this file
+        self.tips_path = Path(__file__).parent / "TIPS.md"
+        self.content = self.tips_path.read_text() if self.tips_path.exists() else ""
+        self._cache = {}
+
+    def _get_libraries(self) -> str:
+        """Extract the libraries section."""
+        if "libraries" not in self._cache:
+            match = re.search(r"## Libraries\n(.*?)(?=\n##)", self.content, re.DOTALL)
+            self._cache["libraries"] = match.group(1).strip() if match else ""
+        return self._cache["libraries"]
+
+    def _get_command_section(self, command: str) -> Optional[str]:
+        """Get the full section for a command."""
+        # Simple and explicit - look for the exact command name
+        # Use negative lookahead to ensure we match ### but not ####
+        pattern = rf"### {re.escape(command)}\n(.*?)(?=\n###(?!#)|\Z)"
+        match = re.search(pattern, self.content, re.DOTALL)
+        return match.group(1).strip() if match else None
+
+    def _get_description(self, command: str) -> Optional[str]:
+        """Get command description (text before #### sections)."""
+        section = self._get_command_section(command)
+        if not section:
+            return None
+
+        # Extract text before first #### heading
+        match = re.match(r"(.*?)(?=\n####|\Z)", section, re.DOTALL)
+        return match.group(1).strip() if match else ""
+
+    def _get_examples(self, command: str) -> Optional[str]:
+        """Get examples section for a command."""
+        section = self._get_command_section(command)
+        if not section:
+            return None
+
+        # Extract Examples section
+        match = re.search(r"#### Examples\n```python\n(.*?)\n```", section, re.DOTALL)
+        return match.group(1).strip() if match else None
+
+    def _get_tips(self, command: str, context: Optional[Dict] = None) -> Optional[List[str]]:
+        """Get tips list for a command."""
+        section = self._get_command_section(command)
+        if not section:
+            return None
+
+        # Extract Tips section
+        match = re.search(r"#### Tips\n(.*?)(?=\n###|\n##|\Z)", section, re.DOTALL)
+        if not match:
+            return None
+
+        tips_text = match.group(1)
+        # Parse bullet points
+        tips = re.findall(r"^- (.+)$", tips_text, re.MULTILINE)
+
+        # Apply context substitutions
+        if context and tips:
+            formatted_tips = []
+            for tip in tips:
+                for key, value in context.items():
+                    tip = tip.replace(f"{{{key}}}", str(value))
+                formatted_tips.append(tip)
+            return formatted_tips
+
+        return tips
+
+    def _get_mcp_description(self, command: str) -> Optional[str]:
+        """Build MCP description from markdown."""
+        description = self._get_description(command)
+        if not description:
+            return None
+
+        # Build complete MCP description
+        parts = [description]
+
+        # Add libraries section for commands with Python expression support
+        if command in ["request", "to_model", "quicktype", "selections"]:
+            parts.append("")
+            parts.append(self._get_libraries())
+
+        # Add examples if available
+        examples = self._get_examples(command)
+        if examples:
+            parts.append("")
+            parts.append("Examples:")
+            # Indent examples
+            for line in examples.split("\n"):
+                parts.append(f"  {line}" if line else "")
+
+        return "\n".join(parts)
+
+
+# Global parser instance
+parser = TipsParser()
+
+
+# Public API
+def get_mcp_description(command: str) -> Optional[str]:
+    """Get MCP description for a command from TIPS.md.
+
+    Args:
+        command: Name of the command.
+    """
+    return parser._get_mcp_description(command)
+
+
+def get_tips(command: str, context: Optional[Dict] = None) -> Optional[List[str]]:
+    """Get developer tips for a command from TIPS.md.
+
+    Args:
+        command: Name of the command.
+        context: Optional context for variable substitution.
+    """
+    return parser._get_tips(command, context)
+
+
+def get_all_tips() -> Dict[str, List[str]]:
+    """Get all available tips from TIPS.md."""
+    all_tips = {}
+
+    # Find all command sections
+    pattern = r"### ([^\n]+)\n"
+    matches = re.findall(pattern, parser.content)
+
+    for command in matches:
+        tips = parser._get_tips(command)
+        if tips:
+            all_tips[command] = tips
+
+    return all_tips

webtap/commands/_utils.py

@@ -0,0 +1,273 @@
+"""Shared utilities for WebTap command modules."""
+
+import ast
+import base64
+import json
+import sys
+from io import StringIO
+from typing import Any, Tuple
+
+
+def evaluate_expression(expr: str, namespace: dict) -> Tuple[Any, str]:
+    """Execute Python code and capture both stdout and the last expression result.
+
+    Args:
+        expr: Python code to execute.
+        namespace: Dict of variables available to the code.
+    """
+    # Standard libraries - always available
+    import re
+    import base64
+    import hashlib
+    import html
+    import urllib.parse
+    import datetime
+    import collections
+    import itertools
+    import pprint
+    import textwrap
+    import difflib
+    import xml.etree.ElementTree as ElementTree
+
+    # Web scraping & parsing
+    from bs4 import BeautifulSoup
+    import lxml.etree
+    import lxml.html
+
+    # Reverse engineering essentials
+    import jwt
+    import yaml
+    import httpx
+    import cryptography.fernet
+    import cryptography.hazmat
+    from google.protobuf import json_format as protobuf_json
+    from google.protobuf import text_format as protobuf_text
+    import msgpack
+
+    # Update namespace with ALL libraries
+    namespace.update(
+        {
+            # Standard
+            "re": re,
+            "json": json,  # Already imported at module level
+            "base64": base64,
+            "hashlib": hashlib,
+            "html": html,
+            "urllib": urllib,
+            "datetime": datetime,
+            "collections": collections,
+            "itertools": itertools,
+            "pprint": pprint,
+            "textwrap": textwrap,
+            "difflib": difflib,
+            "ast": ast,  # Already imported at module level
+            "ElementTree": ElementTree,
+            "ET": ElementTree,  # Common alias
+            # Web scraping
+            "BeautifulSoup": BeautifulSoup,
+            "bs4": BeautifulSoup,  # Alias
+            "lxml": lxml,
+            # Reverse engineering
+            "jwt": jwt,
+            "yaml": yaml,
+            "httpx": httpx,
+            "cryptography": cryptography,
+            "protobuf_json": protobuf_json,
+            "protobuf_text": protobuf_text,
+            "msgpack": msgpack,
+        }
+    )
+
+    # Capture stdout
+    old_stdout = sys.stdout
+    sys.stdout = captured_output = StringIO()
+    result = None
+
+    try:
+        # Parse the code to find if last node is an expression
+        tree = ast.parse(expr)
+        if tree.body:
+            # If last node is an Expression, evaluate it separately
+            if isinstance(tree.body[-1], ast.Expr):
+                # Execute all but the last node
+                if len(tree.body) > 1:
+                    exec_tree = ast.Module(body=tree.body[:-1], type_ignores=[])
+                    exec(compile(exec_tree, "<string>", "exec"), namespace)
+                # Evaluate the last expression
+                result = eval(compile(ast.Expression(body=tree.body[-1].value), "<string>", "eval"), namespace)
+            else:
+                # All statements, just exec everything
+                exec(compile(tree, "<string>", "exec"), namespace)
+
+    except SyntaxError:
+        # Fallback to simple exec if parsing fails
+        exec(expr, namespace)
+    finally:
+        # Always restore stdout
+        sys.stdout = old_stdout
+        output = captured_output.getvalue()
+
+    return result, output
+
+
+def format_expression_result(result: Any, output: str, max_length: int = 2000) -> str:
+    """Format the result of an expression evaluation for display.
+
+    Args:
+        result: The evaluation result.
+        output: Any stdout output captured.
+        max_length: Maximum length before truncation.
+    """
+    parts = []
+
+    if output:
+        parts.append(output.rstrip())
+
+    if result is not None:
+        if isinstance(result, (dict, list)):
+            formatted = json.dumps(result, indent=2)
+            if len(formatted) > max_length:
+                parts.append(formatted[:max_length] + f"\n... [truncated, {len(formatted)} chars total]")
+            else:
+                parts.append(formatted)
+        elif isinstance(result, str) and len(result) > max_length:
+            parts.append(result[:max_length] + f"\n... [truncated, {len(result)} chars total]")
+        else:
+            parts.append(str(result))
+
+    return "\n".join(parts) if parts else "(no output)"
+
+
+# ============= MCP Dict Parameter Utilities =============
+
+
+def parse_options(options: dict | None = None, defaults: dict | None = None) -> dict:
+    """Parse options dict with defaults.
+
+    Args:
+        options: User-provided options dict.
+        defaults: Default values dict.
+    """
+    if defaults is None:
+        defaults = {}
+    if options is None:
+        return defaults.copy()
+
+    result = defaults.copy()
+    result.update(options)
+    return result
+
+
+def extract_option(options: dict | None, key: str, default: object = None, required: bool = False) -> object:
+    """Extract single option from dict with validation.
+
+    Args:
+        options: Options dict to extract from.
+        key: Key to extract.
+        default: Default value if not found.
+        required: Whether the key is required.
+    """
+    if options is None:
+        if required:
+            raise ValueError(f"Required option '{key}' not provided")
+        return default
+
+    if required and key not in options:
+        raise ValueError(f"Required option '{key}' not provided")
+
+    return options.get(key, default)
+
+
+def validate_dict_keys(options: dict | None, allowed: set, required: set | None = None) -> dict:
+    """Validate dict has only allowed keys and all required keys.
+
+    Args:
+        options: Dict to validate.
+        allowed: Set of allowed keys.
+        required: Optional set of required keys.
+    """
+    if options is None:
+        options = {}
+
+    # Check for unknown keys
+    unknown = set(options.keys()) - allowed
+    if unknown:
+        raise ValueError(f"Unknown options: {', '.join(sorted(unknown))}")
+
+    # Check for required keys
+    if required:
+        missing = required - set(options.keys())
+        if missing:
+            raise ValueError(f"Missing required options: {', '.join(sorted(missing))}")
+
+    return options
+
+
+def extract_nested(options: dict | None, path: str, default: object = None) -> object:
+    """Extract nested value from dict using dot notation.
+
+    Args:
+        options: Dict to extract from.
+        path: Dot-separated path.
+        default: Default value if path not found.
+    """
+    if options is None:
+        return default
+
+    current = options
+    for key in path.split("."):
+        if not isinstance(current, dict):
+            return default
+        current = current.get(key)
+        if current is None:
+            return default
+
+    return current
+
+
+# ============= Body Content Utilities =============
+
+
+def fetch_body_content(state, har_entry: dict, field: str) -> tuple[str | None, str | None]:
+    """Fetch body content based on field selector.
+
+    Args:
+        state: WebTap state with client (RPC client).
+        har_entry: HAR entry from request_details().
+        field: Field selector ("response.content" or "request.postData").
+
+    Returns:
+        Tuple of (body_content, error_message).
+    """
+    if field == "response.content":
+        request_id = har_entry.get("request_id")
+        if not request_id:
+            return None, "No request_id in HAR entry"
+
+        try:
+            cdp_result = state.client.call("cdp", command="Network.getResponseBody", params={"requestId": request_id})
+            result = cdp_result.get("result", {})
+        except Exception as e:
+            return None, f"Failed to fetch response body: {e}"
+
+        if not result:
+            return None, "Failed to fetch response body"
+
+        body = result.get("body", "")
+        if result.get("base64Encoded"):
+            try:
+                body = base64.b64decode(body).decode("utf-8")
+            except Exception as e:
+                return None, f"Failed to decode base64 body: {e}"
+
+        return body, None
+
+    elif field == "request.postData":
+        post_data = har_entry.get("request", {}).get("postData", {})
+        text = post_data.get("text")
+        if not text:
+            return None, "No POST data in request"
+        return text, None
+
+    else:
+        return None, f"Unknown field: {field}. Use 'response.content' or 'request.postData'"