webtap-tool 0.6.0__py3-none-any.whl → 0.7.1__py3-none-any.whl

webtap/cdp/session.py

@@ -6,6 +6,7 @@ PUBLIC API:
 
 import json
 import logging
+import queue
 import threading
 from concurrent.futures import Future, TimeoutError
 from typing import Any
@@ -54,9 +55,18 @@ class CDPSession:
         self._lock = threading.Lock()
 
         # DuckDB storage - store events AS-IS
+        # DuckDB connections are NOT thread-safe - use dedicated DB thread
         self.db = duckdb.connect(":memory:")
+        self._db_work_queue: queue.Queue = queue.Queue()
+        self._db_result_queues: dict[int, queue.Queue] = {}
+        self._db_running = True
 
-        self.db.execute("CREATE TABLE events (event JSON)")
+        # Start dedicated database thread
+        self._db_thread = threading.Thread(target=self._db_worker, daemon=True)
+        self._db_thread.start()
+
+        # Initialize schema via queue
+        self._db_execute("CREATE TABLE events (event JSON)", wait_result=False)
 
         # Live field path lookup for fast discovery
         # Maps lowercase field names to their full paths with original case
@@ -71,6 +81,78 @@ class CDPSession:
         self._last_broadcast_time = 0.0
         self._broadcast_debounce = 1.0  # 1 second debounce
 
+    def _db_worker(self) -> None:
+        """Dedicated thread for all database operations.
+
+        Ensures thread safety by serializing all DuckDB access through one thread.
+        DuckDB connections are not thread-safe - sharing them causes malloc corruption.
+        """
+        while self._db_running:
+            try:
+                task = self._db_work_queue.get(timeout=1)
+
+                if task is None:  # Shutdown signal
+                    break
+
+                operation_type, sql, params, result_queue_id = task
+
+                try:
+                    if operation_type == "execute":
+                        result = self.db.execute(sql, params or [])
+                        data = result.fetchall() if result else []
+                    elif operation_type == "delete":
+                        self.db.execute(sql, params or [])
+                        data = None
+                    else:
+                        data = None
+
+                    # Send result back if requested
+                    if result_queue_id and result_queue_id in self._db_result_queues:
+                        self._db_result_queues[result_queue_id].put(("success", data))
+
+                except Exception as e:
+                    logger.error(f"Database error: {e}")
+                    if result_queue_id and result_queue_id in self._db_result_queues:
+                        self._db_result_queues[result_queue_id].put(("error", str(e)))
+
+                finally:
+                    self._db_work_queue.task_done()
+
+            except queue.Empty:
+                continue
+
+    def _db_execute(self, sql: str, params: list | None = None, wait_result: bool = True) -> Any:
+        """Submit database operation to dedicated thread.
+
+        Args:
+            sql: SQL query or command
+            params: Optional query parameters
+            wait_result: Block until operation completes and return result
+
+        Returns:
+            Query results if wait_result=True, None otherwise
+        """
+        result_queue_id = None
+        result_queue = None
+
+        if wait_result:
+            result_queue_id = id(threading.current_thread())
+            result_queue = queue.Queue()
+            self._db_result_queues[result_queue_id] = result_queue
+
+        # Submit to work queue
+        self._db_work_queue.put(("execute", sql, params, result_queue_id))
+
+        if wait_result and result_queue and result_queue_id:
+            status, data = result_queue.get()
+            del self._db_result_queues[result_queue_id]
+
+            if status == "error":
+                raise RuntimeError(f"Database error: {data}")
+            return data
+
+        return None
+
     def list_pages(self) -> list[dict]:
         """List available Chrome pages via HTTP API.
 
@@ -158,6 +240,12 @@ class CDPSession:
             self.ws_thread.join(timeout=2)
             self.ws_thread = None
 
+        # Shutdown database thread
+        self._db_running = False
+        self._db_work_queue.put(None)  # Signal shutdown
+        if self._db_thread.is_alive():
+            self._db_thread.join(timeout=2)
+
         self.connected.clear()
         self.page_info = None
 
@@ -245,7 +333,7 @@ class CDPSession:
 
             # CDP event - store AS-IS in DuckDB and update field lookup
             elif "method" in data:
-                self.db.execute("INSERT INTO events VALUES (?)", [json.dumps(data)])
+                self._db_execute("INSERT INTO events VALUES (?)", [json.dumps(data)], wait_result=False)
                 self._update_field_lookup(data)
 
                 # Call registered event callbacks
@@ -332,7 +420,7 @@ class CDPSession:
 
     def clear_events(self) -> None:
         """Clear all stored events and reset field lookup."""
-        self.db.execute("DELETE FROM events")
+        self._db_execute("DELETE FROM events", wait_result=False)
         self.field_paths.clear()
 
     def query(self, sql: str, params: list | None = None) -> list:
@@ -352,8 +440,7 @@ class CDPSession:
             query("SELECT * FROM events WHERE json_extract_string(event, '$.method') = 'Network.responseReceived'")
             query("SELECT json_extract_string(event, '$.params.request.url') as url FROM events")
         """
-        result = self.db.execute(sql, params or [])
-        return result.fetchall() if result else []
+        return self._db_execute(sql, params)
 
     def fetch_body(self, request_id: str) -> dict | None:
         """Fetch response body via Network.getResponseBody CDP call.

webtap/commands/TIPS.md

@@ -26,7 +26,7 @@ body(123, "msgpack.unpackb(body)")         # Binary formats
 ```
 
 #### Tips
-- **Generate models:** `to_model({id}, "models/model.py")` - create Pydantic models from JSON
+- **Generate models:** `to_model({id}, "models/model.py", "Model")` - create Pydantic models from JSON
 - **Chain requests:** `body({id}, "httpx.get(json.loads(body)['next_url']).text[:100]")`
 - **Parse XML:** `body({id}, "ElementTree.fromstring(body).find('.//title').text")`
 - **Extract forms:** `body({id}, "[f['action'] for f in bs4(body, 'html.parser').find_all('form')]")`
@@ -38,13 +38,14 @@ Generate Pydantic v2 models from JSON response bodies for reverse engineering AP
 
 #### Examples
 ```python
-to_model(123, "models/product.py")                        # Generate from full response
-to_model(123, "models/customer.py", json_path="Data[0]")  # Extract nested object
-to_model(123, "/tmp/model.py", json_path="items")         # Extract array items
+to_model(123, "models/product.py", "Product")                              # Generate from full response
+to_model(123, "models/customers/group.py", "CustomerGroup", "Data[0]")     # Extract nested + domain structure
+to_model(123, "/tmp/item.py", "Item", "items[0]")                          # Extract array items
 ```
 
 #### Tips
 - **Check structure first:** `body({id})` - preview JSON before generating
+- **Domain organization:** Use paths like `"models/customers/group.py"` for structure
 - **Extract nested data:** Use `json_path="Data[0]"` to extract specific objects
 - **Array items:** Extract first item with `json_path="items[0]"` for model generation
 - **Auto-cleanup:** Generated models use snake_case fields and modern type hints (list, dict, | None)
@@ -80,7 +81,7 @@ Show network requests with full data.
 
 #### Tips
 - **Analyze responses:** `body({id})` - fetch response body
-- **Generate models:** `to_model({id}, "models/model.py")` - create Pydantic models from JSON
+- **Generate models:** `to_model({id}, "models/model.py", "Model")` - create Pydantic models from JSON
 - **Parse HTML:** `body({id}, "bs4(body, 'html.parser').find('title').text")`
 - **Extract JSON:** `body({id}, "json.loads(body)['data']")`
 - **Find patterns:** `body({id}, "re.findall(r'/api/\\w+', body)")`

webtap/commands/to_model.py

@@ -12,12 +12,13 @@ mcp_desc = get_mcp_description("to_model")
 
 
 @app.command(display="markdown", fastmcp={"type": "tool", "description": mcp_desc} if mcp_desc else {"type": "tool"})
-def to_model(state, response: int, output: str, json_path: str = None) -> dict:  # pyright: ignore[reportArgumentType]
+def to_model(state, response: int, output: str, model_name: str, json_path: str = None) -> dict:  # pyright: ignore[reportArgumentType]
     """Generate Pydantic model from response body using datamodel-codegen.
 
     Args:
         response: Response row ID from network() table
-        output: Output file path for generated model (e.g., "models/product.py")
+        output: Output file path for generated model (e.g., "models/customers/group.py")
+        model_name: Class name for generated model (e.g., "CustomerGroup")
         json_path: Optional JSON path to extract nested data (e.g., "Data[0]")
 
     Returns:
@@ -102,6 +103,7 @@ def to_model(state, response: int, output: str, json_path: str = None) -> dict:
             input_filename="response.json",
             output=output_path,
             output_model_type=DataModelType.PydanticV2BaseModel,
+            class_name=model_name,  # Set generated class name
             snake_case_field=True,  # Convert to snake_case
             use_standard_collections=True,  # Use list instead of List
             use_union_operator=True,  # Use | instead of Union
@@ -125,5 +127,10 @@ def to_model(state, response: int, output: str, json_path: str = None) -> dict:
 
     return success_response(
         "Model generated successfully",
-        details={"Output": str(output_path), "Fields": field_count, "Size": f"{output_path.stat().st_size} bytes"},
+        details={
+            "Class": model_name,
+            "Output": str(output_path),
+            "Fields": field_count,
+            "Size": f"{output_path.stat().st_size} bytes",
+        },
     )

webtap/services/main.py

@@ -81,8 +81,8 @@ class WebTapService:
         if not self.cdp or not self.cdp.is_connected:
             return 0
         try:
-            result = self.cdp.db.execute("SELECT COUNT(*) FROM events").fetchone()
-            return result[0] if result else 0
+            result = self.cdp.query("SELECT COUNT(*) FROM events")
+            return result[0][0] if result else 0
         except Exception:
             return 0
 

{webtap_tool-0.6.0.dist-info → webtap_tool-0.7.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webtap-tool
-Version: 0.6.0
+Version: 0.7.1
 Summary: Terminal-based web page inspector for AI debugging sessions
 Author-email: Fredrik Angelsen <fredrikangelsen@gmail.com>
 Classifier: Development Status :: 3 - Alpha

{webtap_tool-0.6.0.dist-info → webtap_tool-0.7.1.dist-info}/RECORD

@@ -6,12 +6,12 @@ webtap/filters.py,sha256=kRCicGMSV3R_zSvwzqZqksnry6jxJNdXRcgWvpoBLfc,13323
 webtap/cdp/README.md,sha256=0TS0V_dRgRAzBqhddpXWD4S0YVi5wI4JgFJSll_KUBE,5660
 webtap/cdp/__init__.py,sha256=c6NFG0XJnAa5GTe9MLr9mDZcLZqoTQN7A1cvvOfLcgY,453
 webtap/cdp/query.py,sha256=x2Cy7KMolYkTelpROGezOfMFgYnbSlCvHkvvW1v_gLI,4229
-webtap/cdp/session.py,sha256=LkN676_gig06w9kcXhJXZQF0z0kDZG1TTvispiYw8fQ,16212
+webtap/cdp/session.py,sha256=VAgzcysbRLjhbm9XLMyy1bEmfjQzNYIr-oNhvJosvps,19524
 webtap/cdp/schema/README.md,sha256=hnWCzbXYcYtWaZb_SgjVaFBiG81S9b9Y3x-euQFwQDo,1222
 webtap/cdp/schema/cdp_protocol.json,sha256=dp9_OLYLuVsQb1oV5r6MZfMzURscBLyAXUckdaPWyv4,1488452
 webtap/cdp/schema/cdp_version.json,sha256=OhGy1qpfQjSe3Z7OqL6KynBFlDFBXxKGPZCY-ZN_lVU,399
 webtap/commands/DEVELOPER_GUIDE.md,sha256=LYOhycZ3k5EHx5nREfkjvLz7vOs8pXCRLlcDm-keWao,11973
-webtap/commands/TIPS.md,sha256=9zXJM6VjP23vdODpT6bjWd69uWAnhfs12KapK3Dd8H4,9126
+webtap/commands/TIPS.md,sha256=XwnPKY5AgLsxNw3q0aF4Amr0P891Pt4QIkw0_3AF58g,9293
 webtap/commands/__init__.py,sha256=rr3xM_bY0BgxkDOjsnsI8UBhjlz7nqiYlgJ8fjiJ1jQ,270
 webtap/commands/_builders.py,sha256=SYacZmZTdkolQ7OOf3rFtFPCjkukY8z020WFA-i_O_A,7902
 webtap/commands/_tips.py,sha256=SleMpwdghrHNqdzR60Cu8T0NZqJfWfcfrgIcyWI6GIQ,4793
@@ -30,14 +30,14 @@ webtap/commands/network.py,sha256=gEOg_u7VF9A5aKv5myzLCuvfAUkF1OPxsuj4UAgbS44,31
 webtap/commands/selections.py,sha256=M001d_Gc51aSTuVeXGa19LDh2ZGR_qBJEjVGKpcGGFM,4895
 webtap/commands/server.py,sha256=DOcIgYuKp0ydwrK9EA3hGwqOwfwM9DABhdPu3hk_jjo,6948
 webtap/commands/setup.py,sha256=dov1LaN50nAEMNIuBLSK7mcnwhfn9rtqdTopBm1-PhA,9648
-webtap/commands/to_model.py,sha256=_KVGgptMkYSWgCTA9ekfAneIpwVeCcqIl6f9gwarYUQ,4719
+webtap/commands/to_model.py,sha256=jOb93t616m5weT75VyF506J6nydDXUWENF7cpscbe9Q,4962
 webtap/services/README.md,sha256=rala_jtnNgSiQ1lFLM7x_UQ4SJZDceAm7dpkQMRTYaI,2346
 webtap/services/__init__.py,sha256=IjFqu0Ak6D-r18aokcQMtenDV3fbelvfjTCejGv6CZ0,570
 webtap/services/body.py,sha256=XQPa19y5eUc3XJ2TuwVK6kffO1VQoKqNs33MBBz7hzU,3913
 webtap/services/console.py,sha256=XVfSKTvEHyyOdujsg85S3wtj1CdZhzKtWwlx25MvSv8,3768
 webtap/services/dom.py,sha256=PC-mV56NMLvw37JNzI_jOnZrM7BiDPPn3kOQI9U81vI,19067
 webtap/services/fetch.py,sha256=nl6bpU2Vnf40kau4-mqAnIkhC-7Lx2vbTJKUglz9KnE,13602
-webtap/services/main.py,sha256=KUJFXzXL0ORlYfDV6DNULv_nzNfmlBFz_mfOvscp2wY,6287
+webtap/services/main.py,sha256=5ZUxuIONf4qkKPpKnFi3Au-ZFvuqlpMeDi7fQT_yoQE,6274
 webtap/services/network.py,sha256=0o_--F6YvmXqqFqrcjL1gc6Vr9V1Ytb_U7r_DSUWupA,3444
 webtap/services/setup/__init__.py,sha256=lfoKCAroc-JoE_r7L-KZkF85ZWiB41MBIgrR7ZISSoE,7157
 webtap/services/setup/chrome.py,sha256=zfPWeb6zm_xjIfiS2S_O9lR2BjGKaPXXo06pN_B9lAU,7187
@@ -45,7 +45,7 @@ webtap/services/setup/desktop.py,sha256=fXwQa201W-s2mengm_dJZ9BigJopVrO9YFUQcW_T
 webtap/services/setup/extension.py,sha256=iJY43JlQO6Vicgd9Mz6Mw0LQfbBNUGhnwI8n-LnvHBY,3602
 webtap/services/setup/filters.py,sha256=lAPSLMH_KZQO-7bRkmURwzforx7C3SDrKEw2ZogN-Lo,3220
 webtap/services/setup/platform.py,sha256=7yn-7LQFffgerWzWRtOG-yNEsR36ICThYUAu_N2FAso,4532
-webtap_tool-0.6.0.dist-info/METADATA,sha256=9a8cGzDHRhYnS-aamgu_IW5C3zCtY4pOVmbC-YDZucQ,17636
-webtap_tool-0.6.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-webtap_tool-0.6.0.dist-info/entry_points.txt,sha256=iFe575I0CIb1MbfPt0oX2VYyY5gSU_dA551PKVR83TU,39
-webtap_tool-0.6.0.dist-info/RECORD,,
+webtap_tool-0.7.1.dist-info/METADATA,sha256=Of1VKvajnQxn9m3YdfBTXD2sdQEWCWnzzscaXHCthN4,17636
+webtap_tool-0.7.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+webtap_tool-0.7.1.dist-info/entry_points.txt,sha256=iFe575I0CIb1MbfPt0oX2VYyY5gSU_dA551PKVR83TU,39
+webtap_tool-0.7.1.dist-info/RECORD,,