httpdbg 2.2.1__tar.gz → 2.4.0__tar.gz

{httpdbg-2.2.1 → httpdbg-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: httpdbg
-Version: 2.2.1
+Version: 2.4.0
 Summary: A very simple tool to debug HTTP(S) client and server requests.
 Author-email: cle-b <cle@tictac.pm>
 License-Expression: Apache-2.0
@@ -10,16 +10,16 @@ Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
 Classifier: Topic :: Internet :: WWW/HTTP
 Classifier: Topic :: Software Development :: Testing
 Classifier: Topic :: Utilities
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Dynamic: license-file

{httpdbg-2.2.1 → httpdbg-2.4.0}/httpdbg/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2.2.1"
+__version__ = "2.4.0"
 
 __all__ = ["export_html", "httprecord", "HTTPRecords"]
 

{httpdbg-2.2.1 → httpdbg-2.4.0}/httpdbg/__main__.py

@@ -2,7 +2,6 @@ import argparse
 from pathlib import Path
 import sys
 import time
-from typing import Union
 
 from httpdbg import __version__
 from httpdbg.args import read_args
@@ -32,7 +31,7 @@ def pyhttpdbg(
 
     def run_recorder(
         records: HTTPRecords,
-        initiators: Union[list[str], None],
+        initiators: list[str] | None,
         record_server: bool,
         test_mode: bool,
     ):

{httpdbg-2.2.1 → httpdbg-2.4.0}/httpdbg/export.py

@@ -6,7 +6,6 @@ import xml.etree.ElementTree as ET
 
 import httpdbg
 from httpdbg import HTTPRecords
-from httpdbg.webapp.api import RequestListPayload, RequestPayload
 
 
 def generate_html(records: HTTPRecords, for_export: bool = True) -> str:
@@ -67,12 +66,10 @@ def generate_html(records: HTTPRecords, for_export: bool = True) -> str:
 
     # static export of the requests data
     if for_export:
-        static_all_requests = json.dumps(
-            records, cls=RequestListPayload, ensure_ascii=False
-        )
+        static_all_requests = records.payload.decode("ascii")
         map_requests: dict[str, object] = dict()
         for record in records:
-            map_requests[record.id] = json.loads(json.dumps(record, cls=RequestPayload))
+            map_requests[record.id] = json.loads(record.payload.decode("ascii"))
         static_requests: str = json.dumps(map_requests, ensure_ascii=False)
 
         def safe_for_script_tag(s: str) -> str:

{httpdbg-2.2.1 → httpdbg-2.4.0}/httpdbg/hooks/aiohttp.py

@@ -1,6 +1,6 @@
 from collections.abc import Callable
+from collections.abc import Generator
 from contextlib import contextmanager
-from typing import Generator
 
 from httpdbg.hooks.utils import getcallargs
 from httpdbg.hooks.utils import decorate

{httpdbg-2.2.1 → httpdbg-2.4.0}/httpdbg/hooks/all.py

@@ -1,6 +1,5 @@
+from collections.abc import Generator
 from contextlib import contextmanager
-from typing import Generator
-from typing import Union
 
 from httpdbg.hooks.aiohttp import hook_aiohttp
 from httpdbg.hooks.external import watcher_external
@@ -24,7 +23,7 @@ from httpdbg.records import HTTPRecords
 @contextmanager
 def httprecord(
     records: HTTPRecords = None,
-    initiators: Union[list[str], None] = None,
+    initiators: list[str] | None = None,
     client: bool = True,
     server: bool = False,
     ignore: tuple[tuple[str, int], ...] = (),

{httpdbg-2.2.1 → httpdbg-2.4.0}/httpdbg/hooks/external.py

@@ -1,22 +1,35 @@
+from collections.abc import Generator
 from contextlib import contextmanager
-from typing import Generator
+import json
 import glob
 import os
-import pickle
 import tempfile
 import time
 import threading
-from typing import Union
 
 from httpdbg.env import HTTPDBG_MULTIPROCESS_DIR
+from httpdbg.hooks.recordhttp1 import HTTP1Record
+from httpdbg.hooks.recordhttp2 import HTTP2Record
+from httpdbg.initiator import Group
+from httpdbg.initiator import Initiator
 from httpdbg.log import logger
 from httpdbg.records import HTTPRecords
 
 
+def _safe_delete_file(filename):
+    try:
+        os.remove(filename)
+    except Exception as ex:
+        logger().info(
+            f"_safe_delete_file - silently_catch_error - {str(ex)}",
+            exc_info=True,
+        )
+
+
 @contextmanager
 def watcher_external(
     records: HTTPRecords,
-    initiators: Union[list[str], None] = None,
+    initiators: list[str] | None = None,
     server: bool = False,
 ) -> Generator[HTTPRecords, None, None]:
     if HTTPDBG_MULTIPROCESS_DIR not in os.environ:
@@ -78,12 +91,41 @@ class WatcherSubprocessDirThread(threading.Thread):
         threading.Thread.__init__(self)
 
     def load_dump(self):
-        for dump in glob.glob(os.path.join(self.directory, "*.httpdbgrecords")):
-            with open(dump, "rb") as dumpfile:
-                newrecords: HTTPRecords = pickle.load(dumpfile)
-                self.records.requests.update(newrecords.requests)
-                self.records.initiators.update(newrecords.initiators)
-                self.records.groups.update(newrecords.groups)
+        # initiators
+        for filename in glob.glob(os.path.join(self.directory, "*.httpdbg-initiator")):
+            with open(filename, "r") as f:
+                data = json.load(f)
+                initiator = Initiator.from_json(data)
+                self.records.initiators[initiator.id] = initiator
+            _safe_delete_file(filename)
+
+        # group
+        for filename in glob.glob(os.path.join(self.directory, "*.httpdbg-group")):
+            with open(filename, "r") as f:
+                data = json.load(f)
+                group = Group.from_json(data)
+                self.records.groups[group.id] = group
+            _safe_delete_file(filename)
+
+        # httprecord HTTP1
+        for filename in sorted(
+            glob.glob(os.path.join(self.directory, "*.httpdbg-http1record"))
+        ):
+            with open(filename, "r") as f:
+                data = json.load(f)
+                http1record = HTTP1Record.from_json(data)
+                self.records.requests[http1record.id] = http1record
+            _safe_delete_file(filename)
+
+        # httprecord HTTP2
+        for filename in sorted(
+            glob.glob(os.path.join(self.directory, "*.httpdbg-http2record"))
+        ):
+            with open(filename, "r") as f:
+                data = json.load(f)
+                http2record = HTTP2Record.from_json(data)
+                self.records.requests[http2record.id] = http2record
+            _safe_delete_file(filename)
 
     def run(self):
         while self._running:

{httpdbg-2.2.1 → httpdbg-2.4.0}/httpdbg/hooks/fastapi.py

@@ -1,10 +1,9 @@
 from collections.abc import Callable
+from collections.abc import Generator
 from contextlib import contextmanager
+import datetime
 import functools
 from functools import wraps
-from typing import Generator
-
-import datetime
 
 from httpdbg.hooks.utils import decorate
 from httpdbg.hooks.utils import getcallargs

{httpdbg-2.2.1 → httpdbg-2.4.0}/httpdbg/hooks/flask.py

@@ -1,8 +1,8 @@
 from collections.abc import Callable
+from collections.abc import Generator
 from contextlib import contextmanager
 import functools
 from functools import wraps
-from typing import Generator
 
 
 from httpdbg.hooks.utils import getcallargs

{httpdbg-2.2.1 → httpdbg-2.4.0}/httpdbg/hooks/generic.py

@@ -1,13 +1,12 @@
 import builtins
 from collections.abc import Callable
+from collections.abc import Generator
 from contextlib import contextmanager
 import functools
 import importlib
 import inspect
 from types import ModuleType
 from typing import Any
-from typing import Generator
-from typing import Union
 
 from httpdbg.hooks.utils import decorate
 from httpdbg.hooks.utils import undecorate
@@ -42,7 +41,7 @@ def set_hook_for_generic(records: HTTPRecords, method: Any):
 
 @contextmanager
 def hook_generic(
-    records: HTTPRecords, initiators: Union[list[str], None] = None
+    records: HTTPRecords, initiators: list[str] | None = None
 ) -> Generator[None, None, None]:
     if initiators:
         # we add a hook for a generic initiator only if the module is imported

{httpdbg-2.2.1 → httpdbg-2.4.0}/httpdbg/hooks/h2.py

@@ -1,8 +1,7 @@
 from collections.abc import Callable
+from collections.abc import Generator
 from contextlib import contextmanager
 import random
-from typing import Generator
-from typing import Union
 
 from httpdbg.hooks.recordhttp2 import HTTP2Record
 from httpdbg.hooks.utils import decorate
@@ -67,7 +66,7 @@ class TracerHTTP2:
         stream_id: int,
         headers: list[tuple[bytes, bytes]],
         is_client: bool = True,
-    ) -> Union[HTTP2Record, None]:
+    ) -> HTTP2Record | None:
         record = None
         if not is_client:
             if f"{socket_id}-{stream_id}" in self.sockets:

{httpdbg-2.2.1 → httpdbg-2.4.0}/httpdbg/hooks/http.py

@@ -1,7 +1,6 @@
 from collections.abc import Callable
+from collections.abc import Generator
 from contextlib import contextmanager
-from typing import Generator
-
 
 from httpdbg.hooks.utils import decorate
 from httpdbg.hooks.utils import undecorate

{httpdbg-2.2.1 → httpdbg-2.4.0}/httpdbg/hooks/httpx.py

@@ -1,6 +1,6 @@
 from collections.abc import Callable
+from collections.abc import Generator
 from contextlib import contextmanager
-from typing import Generator
 
 from httpdbg.hooks.utils import getcallargs
 from httpdbg.hooks.utils import decorate

{httpdbg-2.2.1 → httpdbg-2.4.0}/httpdbg/hooks/pytest.py

@@ -1,6 +1,6 @@
 from collections.abc import Callable
+from collections.abc import Generator
 from contextlib import contextmanager
-from typing import Generator
 
 from httpdbg.hooks.utils import decorate
 from httpdbg.hooks.utils import getcallargs

{httpdbg-2.2.1 → httpdbg-2.4.0}/httpdbg/hooks/record.py

@@ -1,6 +1,8 @@
 from abc import ABC, abstractmethod
 import datetime
-from typing import Union
+import json
+import traceback
+from typing import Any
 from urllib.parse import urlparse
 
 from httpdbg.utils import get_new_uuid
@@ -98,12 +100,12 @@ class HTTPRecord(ABC):
     ) -> None:
         self.id = get_new_uuid()
         self.address: tuple[str, int] = ("", 0)
-        self._url: Union[str, None] = None
+        self._url: str | None = None
         self.initiator_id: str = initiator_id
-        self.exception: Union[Exception, None] = None
-        self.ssl: Union[bool, None] = None
+        self.exception: Exception | None = None
+        self.ssl: bool | None = None
         self.tbegin: datetime.datetime = datetime.datetime.now(datetime.timezone.utc)
-        self.tag: Union[str, None] = tag
+        self.tag: str | None = tag
         self.group_id: str = group_id
         self.is_client: bool = is_client
         if tbegin:
@@ -177,3 +179,62 @@ class HTTPRecord(ABC):
             return self.request.protocol
         else:
             return f"{self.request.protocol} -> {self.response.protocol}"
+
+    @property
+    def payload(self) -> bytes:
+        payload: dict[str, Any] = {
+            "id": self.id,
+            "url": self.url,
+            "netloc": self.netloc,
+            "urlext": self.urlext,
+            "method": self.method,
+            "status_code": self.status_code,
+            "http100": self.http100,
+            "reason": self.reason,
+            "request": None,
+            "response": None,
+            "exception": None,
+            "tag": self.tag,
+            "initiator_id": self.initiator_id,
+            "group_id": self.group_id,
+            "in_progress": self.in_progress,
+            "is_server": not self.is_client,
+        }
+
+        payload["request"] = {
+            "headers": [header.to_json() for header in self.request.headers],
+            "cookies": [cookie.to_json() for cookie in self.request.cookies],
+        }
+
+        if self.request.content:
+            payload["request"]["body"] = {
+                "path": f"/request/{self.id}/up",
+                "filename": "upload",
+            }
+            payload["request"]["body"].update(self.request.preview)
+
+        payload["response"] = {
+            "headers": [header.to_json() for header in self.response.headers],
+            "cookies": [cookie.to_json() for cookie in self.response.cookies],
+        }
+
+        if self.response.content:
+            payload["response"]["body"] = {
+                "path": f"/request/{self.id}/down",
+                "filename": "download",
+            }
+            payload["response"]["body"].update(self.response.preview)
+
+        if self.exception is not None:
+            payload["exception"] = {
+                "type": str(type(self.exception)),
+                "message": "".join(
+                    traceback.format_exception(
+                        type(self.exception),
+                        self.exception,
+                        self.exception.__traceback__,
+                    )
+                ),
+            }
+
+        return json.dumps(payload).encode("utf-8")

{httpdbg-2.2.1 → httpdbg-2.4.0}/httpdbg/hooks/recordhttp1.py

@@ -1,3 +1,4 @@
+import base64
 import datetime
 
 from httpdbg.preview import generate_preview
@@ -195,3 +196,34 @@ class HTTP1Record(HTTPRecord):
             self.request.rawdata += data
         else:
             self.response.rawdata += data
+
+    def to_json(self) -> dict:
+        return {
+            "id": self.id,
+            "initiator_id": self.initiator_id,
+            "group_id": self.group_id,
+            "tag": self.tag,
+            "tbegin": self.tbegin.isoformat(),
+            "is_client": self.is_client,
+            "received": base64.b64encode(
+                self.response.rawdata if self.is_client else self.request.rawdata
+            ).decode("ascii"),
+            "sent": base64.b64encode(
+                self.request.rawdata if self.is_client else self.response.rawdata,
+            ).decode("ascii"),
+            "last_update": self.last_update.isoformat(),
+        }
+
+    @staticmethod
+    def from_json(data: dict) -> "HTTP1Record":
+        http1record = HTTP1Record(
+            data["initiator_id"],
+            data["group_id"],
+            data["tag"],
+            datetime.datetime.fromisoformat(data["tbegin"]),
+            data["is_client"],
+        )
+        http1record.id = data["id"]
+        http1record.receive_data(base64.b64decode(data["received"]))
+        http1record.send_data(base64.b64decode(data["sent"]))
+        return http1record

{httpdbg-2.2.1 → httpdbg-2.4.0}/httpdbg/hooks/recordhttp2.py

@@ -1,3 +1,4 @@
+import base64
 import datetime
 
 from httpdbg.preview import generate_preview
@@ -111,3 +112,42 @@ class HTTP2Record(HTTPRecord):
             self.request.content += data
         else:
             self.response.content += data
+
+    def to_json(self) -> dict:
+        return {
+            "id": self.id,
+            "initiator_id": self.initiator_id,
+            "group_id": self.group_id,
+            "tag": self.tag,
+            "tbegin": self.tbegin.isoformat(),
+            "is_client": self.is_client,
+            "request": {
+                "headers": [header.to_json() for header in self.request.headers],
+                "content": base64.b64encode(self.request.content).decode("ascii"),
+            },
+            "response": {
+                "headers": [header.to_json() for header in self.response.headers],
+                "content": base64.b64encode(self.response.content).decode("ascii"),
+            },
+            "last_update": self.last_update.isoformat(),
+        }
+
+    @staticmethod
+    def from_json(data: dict) -> "HTTP2Record":
+        http2record = HTTP2Record(
+            data["initiator_id"],
+            data["group_id"],
+            data["tag"],
+            datetime.datetime.fromisoformat(data["tbegin"]),
+            data["is_client"],
+        )
+        http2record.id = data["id"]
+        http2record.request._headers = [
+            HTTPDBGHeader.from_json(header) for header in data["request"]["headers"]
+        ]
+        http2record.request.content = base64.b64decode(data["request"]["content"])
+        http2record.response._headers = [
+            HTTPDBGHeader.from_json(header) for header in data["response"]["headers"]
+        ]
+        http2record.response.content = base64.b64decode(data["response"]["content"])
+        return http2record

{httpdbg-2.2.1 → httpdbg-2.4.0}/httpdbg/hooks/requests.py

@@ -1,6 +1,6 @@
 from collections.abc import Callable
+from collections.abc import Generator
 from contextlib import contextmanager
-from typing import Generator
 
 from httpdbg.hooks.utils import getcallargs
 from httpdbg.hooks.utils import decorate

{httpdbg-2.2.1 → httpdbg-2.4.0}/httpdbg/hooks/socket.py

@@ -1,14 +1,13 @@
 import asyncio
 import asyncio.proactor_events
 from collections.abc import Callable
+from collections.abc import Generator
 from contextlib import contextmanager
 import datetime
 import platform
 import socket
 import ssl
 import sys
-from typing import Generator
-from typing import Union
 
 from httpdbg.initiator import httpdbg_initiator
 from httpdbg.log import logger
@@ -28,7 +27,7 @@ class SocketRawData(object):
         self.address: tuple[str, int] = address
         self.ssl: bool = ssl
         self._rawdata: bytes = bytes()
-        self.record: Union[HTTP1Record, None] = None
+        self.record: HTTP1Record | None = None
         self.tbegin: datetime.datetime = datetime.datetime.now(datetime.timezone.utc)
 
     @property
@@ -42,7 +41,7 @@ class SocketRawData(object):
         )
         self._rawdata = value
 
-    def http_detected(self) -> Union[bool, None]:
+    def http_detected(self) -> bool | None:
         end_of_first_line = self.rawdata[:2048].find(b"\r\n")
         if end_of_first_line == -1:
             if len(self.rawdata) > 2048:
@@ -66,14 +65,14 @@ class TracerHTTP1:
         self,
         ignore: tuple[tuple[str, int], ...] = (),
     ):
-        self.sockets: dict[int, Union[SocketRawData, None]] = (
+        self.sockets: dict[int, SocketRawData | None] = (
             {}
         )  # if None, this is not a HTTP/1 request
         self.ignore: tuple[tuple[str, int], ...] = ignore
 
     def get_socket_data(
         self, obj, extra_sock=None, force_new=False, request=None, is_uvicorn=False
-    ) -> Union[SocketRawData, None]:
+    ) -> SocketRawData | None:
         """Record a new SocketRawData (or get an existing one) and return it."""
         socketdata = None
 

{httpdbg-2.2.1 → httpdbg-2.4.0}/httpdbg/hooks/starlette.py

@@ -1,7 +1,7 @@
 from collections.abc import Callable
+from collections.abc import Generator
 from contextlib import contextmanager
 from functools import wraps
-from typing import Generator
 
 from httpdbg.hooks.utils import decorate
 from httpdbg.hooks.utils import getcallargs

{httpdbg-2.2.1 → httpdbg-2.4.0}/httpdbg/hooks/template_sitecustomize.py

@@ -1,21 +1,30 @@
 import atexit
+import json
 import os
-import pickle
 import sys
 import time
 import threading
-from typing import Union
 import uuid
 
 from httpdbg import HTTPRecords
 from httpdbg.env import HTTPDBG_MULTIPROCESS_DIR
 from httpdbg.hooks.all import httprecord
+from httpdbg.hooks.record import HTTPRecord
+from httpdbg.hooks.recordhttp1 import HTTP1Record
+from httpdbg.hooks.recordhttp2 import HTTP2Record
 from httpdbg.log import logger
 
 HTTPDBG_INITIATORS = []  # type: ignore
 HTTPDBG_RECORD_SERVER = False
 
 
+def _safe_write_json_file(filename, content):
+    with open(f"{filename}.tmp", "wb") as f:
+        f.write(json.dumps(content).encode("utf-8"))
+
+    os.replace(f"{filename}.tmp", filename)
+
+
 class HttpdbgRecorder:
     """Records the HTTP requests in subprocesses and saves them to disk."""
 
@@ -24,8 +33,7 @@ class HttpdbgRecorder:
         self.fname: str = os.path.join(
             os.environ[HTTPDBG_MULTIPROCESS_DIR], str(uuid.uuid1())
         )
-        self.context = None
-        self.records: Union[HTTPRecords, None] = None
+        self.serialized: list[str] = []
 
     def _save_to_disk_loop(self):
         while self._running:
@@ -45,19 +53,39 @@ class HttpdbgRecorder:
             time.sleep(1)
 
     def save_to_disk(self):
-        if self.records and len(self.records.requests) > 0:
-            with open(f"{self.fname}.httpdbgrecords.tmp", "wb") as f:
-                pickle.dump(self.records, f)
-            os.replace(
-                f"{self.fname}.httpdbgrecords.tmp", f"{self.fname}.httpdbgrecords"
-            )
+        # initiators
+        for initiator in self.records.initiators.values():
+            if initiator.id not in self.serialized:
+                self.serialized.append(initiator.id)
+                filename = f"{self.fname}.{initiator.id}.httpdbg-initiator"
+                _safe_write_json_file(filename, initiator.to_json())
+
+        # groups
+        for group in self.records.groups.values():
+            if group.id not in self.serialized:
+                self.serialized.append(group.id)
+                filename = f"{self.fname}.{group.id}.httpdbg-group"
+                _safe_write_json_file(filename, group.to_json())
+
+        # requests
+        for request in self.records.requests.values():
+            if isinstance(request, HTTPRecord):
+                marker = f"{request.id}-{request.last_update.isoformat()}"
+                if marker not in self.serialized:
+                    self.serialized.append(marker)
+                    if isinstance(request, HTTP1Record):
+                        filename = f"{self.fname}.{marker}.httpdbg-http1record"
+                        _safe_write_json_file(filename, request.to_json())
+                    elif isinstance(request, HTTP2Record):
+                        filename = f"{self.fname}.{marker}.httpdbg-http2record"
+                        _safe_write_json_file(filename, request.to_json())
 
     def start(self):
         """Start recording."""
         self.context = httprecord(
             initiators=HTTPDBG_INITIATORS, server=HTTPDBG_RECORD_SERVER
         )
-        self.records = self.context.__enter__()
+        self.records: HTTPRecords = self.context.__enter__()
         self._running = True
         self._thread = threading.Thread(target=self._save_to_disk_loop, daemon=True)
         self._thread.start()

{httpdbg-2.2.1 → httpdbg-2.4.0}/httpdbg/hooks/unittest.py

@@ -1,9 +1,9 @@
 from collections.abc import Callable
+from collections.abc import Generator
 from contextlib import contextmanager
 import inspect
 import os
 import typing
-from typing import Generator
 
 from httpdbg.initiator import httpdbg_group
 from httpdbg.initiator import httpdbg_tag

{httpdbg-2.2.1 → httpdbg-2.4.0}/httpdbg/hooks/urllib3.py

@@ -1,6 +1,6 @@
 from collections.abc import Callable
+from collections.abc import Generator
 from contextlib import contextmanager
-from typing import Generator
 
 from httpdbg.hooks.utils import getcallargs
 from httpdbg.hooks.utils import decorate

{httpdbg-2.2.1 → httpdbg-2.4.0}/httpdbg/hooks/uvicorn.py

@@ -1,8 +1,8 @@
 from collections.abc import Callable
+from collections.abc import Generator
 from contextlib import contextmanager
 import datetime
 import typing
-from typing import Generator
 
 from httpdbg.hooks.utils import decorate
 from httpdbg.hooks.utils import undecorate