webtap-tool 0.5.2__tar.gz → 0.6.0__tar.gz

{webtap_tool-0.5.2 → webtap_tool-0.6.0}/CHANGELOG.md

@@ -15,6 +15,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Removed
 
+## [0.6.0] - 2025-10-10
+
+### Added
+- **to_model command**: Generate Pydantic v2 models from JSON response bodies using datamodel-codegen
+  - Supports `json_path` parameter for extracting nested objects (e.g., `"Data[0]"`)
+  - Auto-applies snake_case fields and modern type hints (`list`, `dict`, `| None`)
+  - Perfect for reverse engineering APIs - inspect with `body()`, generate models with `to_model()`
+- **datamodel-code-generator dependency**: Added for Python-native model generation (no subprocess)
+
+### Changed
+- **TIPS.md**: Added `to_model` cross-references in `body` and `network` tips
+
+### Fixed
+
+### Removed
+
 ## [0.5.2] - 2025-10-09
 
 ### Added

{webtap_tool-0.5.2 → webtap_tool-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webtap-tool
-Version: 0.5.2
+Version: 0.6.0
 Summary: Terminal-based web page inspector for AI debugging sessions
 Author-email: Fredrik Angelsen <fredrikangelsen@gmail.com>
 Classifier: Development Status :: 3 - Alpha
@@ -12,6 +12,7 @@ Classifier: Topic :: Software Development :: Debuggers
 Requires-Python: >=3.12
 Requires-Dist: beautifulsoup4>=4.13.5
 Requires-Dist: cryptography>=45.0.6
+Requires-Dist: datamodel-code-generator>=0.35.0
 Requires-Dist: duckdb>=1.3.2
 Requires-Dist: fastapi>=0.116.1
 Requires-Dist: httpx>=0.28.1

{webtap_tool-0.5.2 → webtap_tool-0.6.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "webtap-tool"
-version = "0.5.2"
+version = "0.6.0"
 description = "Terminal-based web page inspector for AI debugging sessions"
 readme = "README.md"
 authors = [
@@ -18,6 +18,7 @@ classifiers = [
 dependencies = [
     "beautifulsoup4>=4.13.5",
     "cryptography>=45.0.6",
+    "datamodel-code-generator>=0.35.0",
     "duckdb>=1.3.2",
     "fastapi>=0.116.1",
     "httpx>=0.28.1",

{webtap_tool-0.5.2 → webtap_tool-0.6.0}/src/webtap/app.py

@@ -89,6 +89,7 @@ else:
     from webtap.commands import inspect  # noqa: E402, F401
     from webtap.commands import fetch  # noqa: E402, F401
     from webtap.commands import body  # noqa: E402, F401
+    from webtap.commands import to_model  # noqa: E402, F401
     from webtap.commands import selections  # noqa: E402, F401
     from webtap.commands import server  # noqa: E402, F401
     from webtap.commands import setup  # noqa: E402, F401

{webtap_tool-0.5.2 → webtap_tool-0.6.0}/src/webtap/commands/TIPS.md

@@ -26,12 +26,30 @@ body(123, "msgpack.unpackb(body)")         # Binary formats
 ```
 
 #### Tips
+- **Generate models:** `to_model({id}, "models/model.py")` - create Pydantic models from JSON
 - **Chain requests:** `body({id}, "httpx.get(json.loads(body)['next_url']).text[:100]")`
 - **Parse XML:** `body({id}, "ElementTree.fromstring(body).find('.//title').text")`
 - **Extract forms:** `body({id}, "[f['action'] for f in bs4(body, 'html.parser').find_all('form')]")`
 - **Decode protobuf:** `body({id}, "protobuf_json.Parse(body, YourMessage())")`
 - **Find related:** `events({'requestId': request_id})` - related events
 
+### to_model
+Generate Pydantic v2 models from JSON response bodies for reverse engineering APIs.
+
+#### Examples
+```python
+to_model(123, "models/product.py")                        # Generate from full response
+to_model(123, "models/customer.py", json_path="Data[0]")  # Extract nested object
+to_model(123, "/tmp/model.py", json_path="items")         # Extract array items
+```
+
+#### Tips
+- **Check structure first:** `body({id})` - preview JSON before generating
+- **Extract nested data:** Use `json_path="Data[0]"` to extract specific objects
+- **Array items:** Extract first item with `json_path="items[0]"` for model generation
+- **Auto-cleanup:** Generated models use snake_case fields and modern type hints (list, dict, | None)
+- **Edit after:** Add `Field(alias="...")` manually for API field mapping
+
 ### inspect
 Inspect CDP events with full Python debugging.
 
@@ -62,6 +80,7 @@ Show network requests with full data.
 
 #### Tips
 - **Analyze responses:** `body({id})` - fetch response body
+- **Generate models:** `to_model({id}, "models/model.py")` - create Pydantic models from JSON
 - **Parse HTML:** `body({id}, "bs4(body, 'html.parser').find('title').text")`
 - **Extract JSON:** `body({id}, "json.loads(body)['data']")`
 - **Find patterns:** `body({id}, "re.findall(r'/api/\\w+', body)")`

webtap_tool-0.6.0/src/webtap/commands/to_model.py

@@ -0,0 +1,129 @@
+"""Generate Pydantic models from HTTP response bodies."""
+
+import json
+from pathlib import Path
+from datamodel_code_generator import generate, InputFileType, DataModelType
+from webtap.app import app
+from webtap.commands._builders import check_connection, success_response, error_response
+from webtap.commands._tips import get_mcp_description
+
+
+mcp_desc = get_mcp_description("to_model")
+
+
+@app.command(display="markdown", fastmcp={"type": "tool", "description": mcp_desc} if mcp_desc else {"type": "tool"})
+def to_model(state, response: int, output: str, json_path: str = None) -> dict:  # pyright: ignore[reportArgumentType]
+    """Generate Pydantic model from response body using datamodel-codegen.
+
+    Args:
+        response: Response row ID from network() table
+        output: Output file path for generated model (e.g., "models/product.py")
+        json_path: Optional JSON path to extract nested data (e.g., "Data[0]")
+
+    Returns:
+        Success message with generation details
+    """
+    if error := check_connection(state):
+        return error
+
+    # Get response body from service
+    body_service = state.service.body
+    result = body_service.get_response_body(response, use_cache=True)
+
+    if "error" in result:
+        return error_response(result["error"])
+
+    body_content = result.get("body", "")
+    is_base64 = result.get("base64Encoded", False)
+
+    # Decode if needed
+    if is_base64:
+        decoded = body_service.decode_body(body_content, is_base64)
+        if isinstance(decoded, bytes):
+            return error_response(
+                "Response body is binary",
+                suggestions=["Only JSON responses can be converted to models", "Try a different response"],
+            )
+        body_content = decoded
+
+    # Parse JSON
+    try:
+        data = json.loads(body_content)
+    except json.JSONDecodeError as e:
+        return error_response(
+            f"Invalid JSON: {e}",
+            suggestions=[
+                "Response must be valid JSON",
+                "Check the response with body() first",
+                "Try a different response",
+            ],
+        )
+
+    # Extract JSON path if specified
+    if json_path:
+        try:
+            # Support simple bracket notation like "Data[0]"
+            parts = json_path.replace("[", ".").replace("]", "").split(".")
+            for part in parts:
+                if part:
+                    if part.isdigit():
+                        data = data[int(part)]
+                    else:
+                        data = data[part]
+        except (KeyError, IndexError, TypeError) as e:
+            return error_response(
+                f"JSON path extraction failed: {e}",
+                suggestions=[
+                    f"Path '{json_path}' not found in response",
+                    "Check the response structure with body()",
+                    'Try a simpler path like "Data" or "Data[0]"',
+                ],
+            )
+
+    # Ensure data is dict or list for model generation
+    if not isinstance(data, (dict, list)):
+        return error_response(
+            f"Extracted data is {type(data).__name__}, not dict or list",
+            suggestions=[
+                "Model generation requires dict or list structure",
+                "Adjust json_path to extract a complex object",
+            ],
+        )
+
+    # Create output directory if needed
+    output_path = Path(output).expanduser().resolve()
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # Generate model using datamodel-codegen Python API
+    try:
+        generate(
+            json.dumps(data),
+            input_file_type=InputFileType.Json,
+            input_filename="response.json",
+            output=output_path,
+            output_model_type=DataModelType.PydanticV2BaseModel,
+            snake_case_field=True,  # Convert to snake_case
+            use_standard_collections=True,  # Use list instead of List
+            use_union_operator=True,  # Use | instead of Union
+        )
+    except Exception as e:
+        return error_response(
+            f"Model generation failed: {e}",
+            suggestions=[
+                "Check that the JSON structure is valid",
+                "Try simplifying the JSON path",
+                "Ensure output directory is writable",
+            ],
+        )
+
+    # Count fields in generated model
+    try:
+        model_content = output_path.read_text()
+        field_count = model_content.count(": ")  # Count field definitions
+    except Exception:
+        field_count = "unknown"
+
+    return success_response(
+        "Model generated successfully",
+        details={"Output": str(output_path), "Fields": field_count, "Size": f"{output_path.stat().st_size} bytes"},
+    )