httpdbg 2.3.0__tar.gz → 2.4.0__tar.gz

{httpdbg-2.3.0 → httpdbg-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: httpdbg
-Version: 2.3.0
+Version: 2.4.0
 Summary: A very simple tool to debug HTTP(S) client and server requests.
 Author-email: cle-b <cle@tictac.pm>
 License-Expression: Apache-2.0

{httpdbg-2.3.0 → httpdbg-2.4.0}/httpdbg/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2.3.0"
+__version__ = "2.4.0"
 
 __all__ = ["export_html", "httprecord", "HTTPRecords"]
 

{httpdbg-2.3.0 → httpdbg-2.4.0}/httpdbg/export.py

@@ -6,7 +6,6 @@ import xml.etree.ElementTree as ET
 
 import httpdbg
 from httpdbg import HTTPRecords
-from httpdbg.webapp.api import RequestListPayload, RequestPayload
 
 
 def generate_html(records: HTTPRecords, for_export: bool = True) -> str:
@@ -67,12 +66,10 @@ def generate_html(records: HTTPRecords, for_export: bool = True) -> str:
 
     # static export of the requests data
     if for_export:
-        static_all_requests = json.dumps(
-            records, cls=RequestListPayload, ensure_ascii=False
-        )
+        static_all_requests = records.payload.decode("ascii")
         map_requests: dict[str, object] = dict()
         for record in records:
-            map_requests[record.id] = json.loads(json.dumps(record, cls=RequestPayload))
+            map_requests[record.id] = json.loads(record.payload.decode("ascii"))
         static_requests: str = json.dumps(map_requests, ensure_ascii=False)
 
         def safe_for_script_tag(s: str) -> str:

{httpdbg-2.3.0 → httpdbg-2.4.0}/httpdbg/hooks/external.py

@@ -1,17 +1,31 @@
 from collections.abc import Generator
 from contextlib import contextmanager
+import json
 import glob
 import os
-import pickle
 import tempfile
 import time
 import threading
 
 from httpdbg.env import HTTPDBG_MULTIPROCESS_DIR
+from httpdbg.hooks.recordhttp1 import HTTP1Record
+from httpdbg.hooks.recordhttp2 import HTTP2Record
+from httpdbg.initiator import Group
+from httpdbg.initiator import Initiator
 from httpdbg.log import logger
 from httpdbg.records import HTTPRecords
 
 
+def _safe_delete_file(filename):
+    try:
+        os.remove(filename)
+    except Exception as ex:
+        logger().info(
+            f"_safe_delete_file - silently_catch_error - {str(ex)}",
+            exc_info=True,
+        )
+
+
 @contextmanager
 def watcher_external(
     records: HTTPRecords,
@@ -77,12 +91,41 @@ class WatcherSubprocessDirThread(threading.Thread):
         threading.Thread.__init__(self)
 
     def load_dump(self):
-        for dump in glob.glob(os.path.join(self.directory, "*.httpdbgrecords")):
-            with open(dump, "rb") as dumpfile:
-                newrecords: HTTPRecords = pickle.load(dumpfile)
-                self.records.requests.update(newrecords.requests)
-                self.records.initiators.update(newrecords.initiators)
-                self.records.groups.update(newrecords.groups)
+        # initiators
+        for filename in glob.glob(os.path.join(self.directory, "*.httpdbg-initiator")):
+            with open(filename, "r") as f:
+                data = json.load(f)
+                initiator = Initiator.from_json(data)
+                self.records.initiators[initiator.id] = initiator
+            _safe_delete_file(filename)
+
+        # group
+        for filename in glob.glob(os.path.join(self.directory, "*.httpdbg-group")):
+            with open(filename, "r") as f:
+                data = json.load(f)
+                group = Group.from_json(data)
+                self.records.groups[group.id] = group
+            _safe_delete_file(filename)
+
+        # httprecord HTTP1
+        for filename in sorted(
+            glob.glob(os.path.join(self.directory, "*.httpdbg-http1record"))
+        ):
+            with open(filename, "r") as f:
+                data = json.load(f)
+                http1record = HTTP1Record.from_json(data)
+                self.records.requests[http1record.id] = http1record
+            _safe_delete_file(filename)
+
+        # httprecord HTTP2
+        for filename in sorted(
+            glob.glob(os.path.join(self.directory, "*.httpdbg-http2record"))
+        ):
+            with open(filename, "r") as f:
+                data = json.load(f)
+                http2record = HTTP2Record.from_json(data)
+                self.records.requests[http2record.id] = http2record
+            _safe_delete_file(filename)
 
     def run(self):
         while self._running:

{httpdbg-2.3.0 → httpdbg-2.4.0}/httpdbg/hooks/record.py

@@ -1,5 +1,8 @@
 from abc import ABC, abstractmethod
 import datetime
+import json
+import traceback
+from typing import Any
 from urllib.parse import urlparse
 
 from httpdbg.utils import get_new_uuid
@@ -176,3 +179,62 @@ class HTTPRecord(ABC):
             return self.request.protocol
         else:
             return f"{self.request.protocol} -> {self.response.protocol}"
+
+    @property
+    def payload(self) -> bytes:
+        payload: dict[str, Any] = {
+            "id": self.id,
+            "url": self.url,
+            "netloc": self.netloc,
+            "urlext": self.urlext,
+            "method": self.method,
+            "status_code": self.status_code,
+            "http100": self.http100,
+            "reason": self.reason,
+            "request": None,
+            "response": None,
+            "exception": None,
+            "tag": self.tag,
+            "initiator_id": self.initiator_id,
+            "group_id": self.group_id,
+            "in_progress": self.in_progress,
+            "is_server": not self.is_client,
+        }
+
+        payload["request"] = {
+            "headers": [header.to_json() for header in self.request.headers],
+            "cookies": [cookie.to_json() for cookie in self.request.cookies],
+        }
+
+        if self.request.content:
+            payload["request"]["body"] = {
+                "path": f"/request/{self.id}/up",
+                "filename": "upload",
+            }
+            payload["request"]["body"].update(self.request.preview)
+
+        payload["response"] = {
+            "headers": [header.to_json() for header in self.response.headers],
+            "cookies": [cookie.to_json() for cookie in self.response.cookies],
+        }
+
+        if self.response.content:
+            payload["response"]["body"] = {
+                "path": f"/request/{self.id}/down",
+                "filename": "download",
+            }
+            payload["response"]["body"].update(self.response.preview)
+
+        if self.exception is not None:
+            payload["exception"] = {
+                "type": str(type(self.exception)),
+                "message": "".join(
+                    traceback.format_exception(
+                        type(self.exception),
+                        self.exception,
+                        self.exception.__traceback__,
+                    )
+                ),
+            }
+
+        return json.dumps(payload).encode("utf-8")

{httpdbg-2.3.0 → httpdbg-2.4.0}/httpdbg/hooks/recordhttp1.py

@@ -1,3 +1,4 @@
+import base64
 import datetime
 
 from httpdbg.preview import generate_preview
@@ -195,3 +196,34 @@ class HTTP1Record(HTTPRecord):
             self.request.rawdata += data
         else:
             self.response.rawdata += data
+
+    def to_json(self) -> dict:
+        return {
+            "id": self.id,
+            "initiator_id": self.initiator_id,
+            "group_id": self.group_id,
+            "tag": self.tag,
+            "tbegin": self.tbegin.isoformat(),
+            "is_client": self.is_client,
+            "received": base64.b64encode(
+                self.response.rawdata if self.is_client else self.request.rawdata
+            ).decode("ascii"),
+            "sent": base64.b64encode(
+                self.request.rawdata if self.is_client else self.response.rawdata,
+            ).decode("ascii"),
+            "last_update": self.last_update.isoformat(),
+        }
+
+    @staticmethod
+    def from_json(data: dict) -> "HTTP1Record":
+        http1record = HTTP1Record(
+            data["initiator_id"],
+            data["group_id"],
+            data["tag"],
+            datetime.datetime.fromisoformat(data["tbegin"]),
+            data["is_client"],
+        )
+        http1record.id = data["id"]
+        http1record.receive_data(base64.b64decode(data["received"]))
+        http1record.send_data(base64.b64decode(data["sent"]))
+        return http1record

{httpdbg-2.3.0 → httpdbg-2.4.0}/httpdbg/hooks/recordhttp2.py

@@ -1,3 +1,4 @@
+import base64
 import datetime
 
 from httpdbg.preview import generate_preview
@@ -111,3 +112,42 @@ class HTTP2Record(HTTPRecord):
             self.request.content += data
         else:
             self.response.content += data
+
+    def to_json(self) -> dict:
+        return {
+            "id": self.id,
+            "initiator_id": self.initiator_id,
+            "group_id": self.group_id,
+            "tag": self.tag,
+            "tbegin": self.tbegin.isoformat(),
+            "is_client": self.is_client,
+            "request": {
+                "headers": [header.to_json() for header in self.request.headers],
+                "content": base64.b64encode(self.request.content).decode("ascii"),
+            },
+            "response": {
+                "headers": [header.to_json() for header in self.response.headers],
+                "content": base64.b64encode(self.response.content).decode("ascii"),
+            },
+            "last_update": self.last_update.isoformat(),
+        }
+
+    @staticmethod
+    def from_json(data: dict) -> "HTTP2Record":
+        http2record = HTTP2Record(
+            data["initiator_id"],
+            data["group_id"],
+            data["tag"],
+            datetime.datetime.fromisoformat(data["tbegin"]),
+            data["is_client"],
+        )
+        http2record.id = data["id"]
+        http2record.request._headers = [
+            HTTPDBGHeader.from_json(header) for header in data["request"]["headers"]
+        ]
+        http2record.request.content = base64.b64decode(data["request"]["content"])
+        http2record.response._headers = [
+            HTTPDBGHeader.from_json(header) for header in data["response"]["headers"]
+        ]
+        http2record.response.content = base64.b64decode(data["response"]["content"])
+        return http2record

{httpdbg-2.3.0 → httpdbg-2.4.0}/httpdbg/hooks/template_sitecustomize.py

@@ -1,6 +1,6 @@
 import atexit
+import json
 import os
-import pickle
 import sys
 import time
 import threading
@@ -9,12 +9,22 @@ import uuid
 from httpdbg import HTTPRecords
 from httpdbg.env import HTTPDBG_MULTIPROCESS_DIR
 from httpdbg.hooks.all import httprecord
+from httpdbg.hooks.record import HTTPRecord
+from httpdbg.hooks.recordhttp1 import HTTP1Record
+from httpdbg.hooks.recordhttp2 import HTTP2Record
 from httpdbg.log import logger
 
 HTTPDBG_INITIATORS = []  # type: ignore
 HTTPDBG_RECORD_SERVER = False
 
 
+def _safe_write_json_file(filename, content):
+    with open(f"{filename}.tmp", "wb") as f:
+        f.write(json.dumps(content).encode("utf-8"))
+
+    os.replace(f"{filename}.tmp", filename)
+
+
 class HttpdbgRecorder:
     """Records the HTTP requests in subprocesses and saves them to disk."""
 
@@ -23,8 +33,7 @@ class HttpdbgRecorder:
         self.fname: str = os.path.join(
             os.environ[HTTPDBG_MULTIPROCESS_DIR], str(uuid.uuid1())
         )
-        self.context = None
-        self.records: HTTPRecords | None = None
+        self.serialized: list[str] = []
 
     def _save_to_disk_loop(self):
         while self._running:
@@ -44,19 +53,39 @@ class HttpdbgRecorder:
             time.sleep(1)
 
     def save_to_disk(self):
-        if self.records and len(self.records.requests) > 0:
-            with open(f"{self.fname}.httpdbgrecords.tmp", "wb") as f:
-                pickle.dump(self.records, f)
-            os.replace(
-                f"{self.fname}.httpdbgrecords.tmp", f"{self.fname}.httpdbgrecords"
-            )
+        # initiators
+        for initiator in self.records.initiators.values():
+            if initiator.id not in self.serialized:
+                self.serialized.append(initiator.id)
+                filename = f"{self.fname}.{initiator.id}.httpdbg-initiator"
+                _safe_write_json_file(filename, initiator.to_json())
+
+        # groups
+        for group in self.records.groups.values():
+            if group.id not in self.serialized:
+                self.serialized.append(group.id)
+                filename = f"{self.fname}.{group.id}.httpdbg-group"
+                _safe_write_json_file(filename, group.to_json())
+
+        # requests
+        for request in self.records.requests.values():
+            if isinstance(request, HTTPRecord):
+                marker = f"{request.id}-{request.last_update.isoformat()}"
+                if marker not in self.serialized:
+                    self.serialized.append(marker)
+                    if isinstance(request, HTTP1Record):
+                        filename = f"{self.fname}.{marker}.httpdbg-http1record"
+                        _safe_write_json_file(filename, request.to_json())
+                    elif isinstance(request, HTTP2Record):
+                        filename = f"{self.fname}.{marker}.httpdbg-http2record"
+                        _safe_write_json_file(filename, request.to_json())
 
     def start(self):
         """Start recording."""
         self.context = httprecord(
             initiators=HTTPDBG_INITIATORS, server=HTTPDBG_RECORD_SERVER
         )
-        self.records = self.context.__enter__()
+        self.records: HTTPRecords = self.context.__enter__()
         self._running = True
         self._thread = threading.Thread(target=self._save_to_disk_loop, daemon=True)
         self._thread.start()

{httpdbg-2.3.0 → httpdbg-2.4.0}/httpdbg/initiator.py

@@ -59,6 +59,17 @@ class Initiator:
             }
         return json
 
+    @staticmethod
+    def from_json(data: dict) -> "Initiator":
+        initiator = Initiator(
+            data["label"],
+            data["short_stack"],
+            data["stack"].split("\n"),
+        )
+        initiator.id = data["id"]
+        initiator.tbegin = datetime.datetime.fromisoformat(data["tbegin"])
+        return initiator
+
 
 class Group:
     def __init__(self, label: str, full_label: str, updatable: bool):
@@ -76,6 +87,17 @@ class Group:
             "tbegin": self.tbegin.isoformat(),
         }
 
+    @staticmethod
+    def from_json(data: dict) -> "Group":
+        group = Group(
+            data["label"],
+            data["full_label"],
+            False,
+        )
+        group.id = data["id"]
+        group.tbegin = datetime.datetime.fromisoformat(data["tbegin"])
+        return group
+
 
 def compatible_path(path: str) -> str:
     p = path

{httpdbg-2.3.0 → httpdbg-2.4.0}/httpdbg/records.py

@@ -1,4 +1,5 @@
 import datetime
+import json
 import sys
 
 from httpdbg.hooks.record import HTTPRecord
@@ -89,3 +90,44 @@ class HTTPRecords:
             print(f"+ {request.url}")
             print(f"  - initiator: {self.initiators[request.initiator_id].label}")
             print(f"  - group: {self.groups[request.group_id].label}")
+
+    @property
+    def payload(self) -> bytes:
+        payload: dict[str, dict[str, str | dict]] = {
+            "session": {
+                "id": self.session.id,
+                "command_line": self.session.command_line,
+                "tbegin": self.session.tbegin.isoformat(),
+            },
+            "requests": {},
+            "initiators": {},
+            "groups": {},
+        }
+
+        for id, req in self.requests.items():
+            payload["requests"][id] = {
+                "id": req.id,
+                "url": req.url,
+                "netloc": req.netloc,
+                "urlext": req.urlext,
+                "status_code": req.status_code,
+                "http100": req.http100,
+                "protocol": req.protocol,
+                "reason": req.reason,
+                "verb": req.method,
+                "tag": req.tag,
+                "initiator_id": req.initiator_id,
+                "group_id": req.group_id,
+                "in_progress": req.in_progress,
+                "is_server": not req.is_client,
+                "tbegin": req.tbegin.isoformat(),
+                "last_update": req.last_update.isoformat(),
+            }
+
+        for id, initiator in self.initiators.items():
+            payload["initiators"][id] = initiator.to_json(full=True)
+
+        for id, group in self.groups.items():
+            payload["groups"][id] = group.to_json()
+
+        return json.dumps(payload).encode("utf-8")

{httpdbg-2.3.0 → httpdbg-2.4.0}/httpdbg/utils.py

@@ -66,6 +66,10 @@ class HTTPDBGHeader(object):
     def to_json(self) -> dict:
         return {"name": self.name, "value": self.value}
 
+    @staticmethod
+    def from_json(data) -> "HTTPDBGHeader":
+        return HTTPDBGHeader(data["name"], data["value"])
+
     def __eq__(self, other) -> bool:
         if type(other) is HTTPDBGHeader:
             return (self.name == other.name) and (self.value == other.value)

{httpdbg-2.3.0 → httpdbg-2.4.0}/httpdbg/webapp/app.py

@@ -1,7 +1,6 @@
 from collections.abc import Callable
 from contextlib import contextmanager
 from http.server import BaseHTTPRequestHandler
-import json
 from pathlib import Path
 import re
 from urllib.parse import ParseResult
@@ -9,7 +8,6 @@ from urllib.parse import parse_qs
 from urllib.parse import urlparse
 
 from httpdbg.log import logger
-from httpdbg.webapp.api import RequestListPayload, RequestPayload
 from httpdbg.records import HTTPRecords
 
 
@@ -97,9 +95,7 @@ class HttpbgHTTPRequestHandler(BaseHTTPRequestHandler):
         self.send_header("Content-type", "application/json")
         self.send_header_no_cache()
         self.end_headers()
-        self.wfile.write(
-            json.dumps(self.records, cls=RequestListPayload).encode("utf-8")
-        )
+        self.wfile.write(self.records.payload)
 
         return True
 
@@ -118,11 +114,7 @@ class HttpbgHTTPRequestHandler(BaseHTTPRequestHandler):
         self.send_header("Content-type", "application/json")
         self.send_header_no_cache()
         self.end_headers()
-        self.wfile.write(
-            json.dumps(self.records.requests[req_id], cls=RequestPayload).encode(
-                "utf-8"
-            )
-        )
+        self.wfile.write(self.records.requests[req_id].payload)
 
         return True
 

{httpdbg-2.3.0 → httpdbg-2.4.0}/httpdbg.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: httpdbg
-Version: 2.3.0
+Version: 2.4.0
 Summary: A very simple tool to debug HTTP(S) client and server requests.
 Author-email: cle-b <cle@tictac.pm>
 License-Expression: Apache-2.0

{httpdbg-2.3.0 → httpdbg-2.4.0}/httpdbg.egg-info/SOURCES.txt

@@ -46,7 +46,6 @@ httpdbg/hooks/urllib3.py
 httpdbg/hooks/utils.py
 httpdbg/hooks/uvicorn.py
 httpdbg/webapp/__init__.py
-httpdbg/webapp/api.py
 httpdbg/webapp/app.py
 httpdbg/webapp/static/api.js
 httpdbg/webapp/static/column.css

httpdbg-2.3.0/httpdbg/webapp/api.py

@@ -1,110 +0,0 @@
-from json import JSONEncoder
-import traceback
-from typing import Any
-
-from httpdbg.records import HTTPRecords
-from httpdbg.records import HTTPRecord
-
-
-class RequestPayload(JSONEncoder):
-    def default(self, req: HTTPRecord) -> dict[str, Any]:
-        payload: dict[str, Any] = {
-            "id": req.id,
-            "url": req.url,
-            "netloc": req.netloc,
-            "urlext": req.urlext,
-            "method": req.method,
-            "status_code": req.status_code,
-            "http100": req.http100,
-            "reason": req.reason,
-            "request": None,
-            "response": None,
-            "exception": None,
-            "tag": req.tag,
-            "initiator_id": req.initiator_id,
-            "group_id": req.group_id,
-            "in_progress": req.in_progress,
-            "is_server": not req.is_client,
-        }
-
-        payload["request"] = {
-            "headers": [header.to_json() for header in req.request.headers],
-            "cookies": [cookie.to_json() for cookie in req.request.cookies],
-        }
-
-        if req.request.content:
-            payload["request"]["body"] = {
-                "path": f"/request/{req.id}/up",
-                "filename": "upload",
-            }
-            payload["request"]["body"].update(req.request.preview)
-
-        payload["response"] = {
-            "headers": [header.to_json() for header in req.response.headers],
-            "cookies": [cookie.to_json() for cookie in req.response.cookies],
-        }
-
-        if req.response.content:
-            payload["response"]["body"] = {
-                "path": f"/request/{req.id}/down",
-                "filename": "download",
-            }
-            payload["response"]["body"].update(req.response.preview)
-
-        if req.exception is not None:
-            payload["exception"] = {
-                "type": str(type(req.exception)),
-                "message": "".join(
-                    traceback.format_exception(
-                        type(req.exception), req.exception, req.exception.__traceback__
-                    )
-                ),
-            }
-
-        return payload
-
-
-class RequestListPayload(JSONEncoder):
-    def default(self, records: HTTPRecords):
-        assert isinstance(
-            records, HTTPRecords
-        ), "This encoder works only for HTTPRecords object."
-
-        payload: dict[str, dict[str, str | dict]] = {
-            "session": {
-                "id": records.session.id,
-                "command_line": records.session.command_line,
-                "tbegin": records.session.tbegin.isoformat(),
-            },
-            "requests": {},
-            "initiators": {},
-            "groups": {},
-        }
-
-        for id, req in records.requests.items():
-            payload["requests"][id] = {
-                "id": req.id,
-                "url": req.url,
-                "netloc": req.netloc,
-                "urlext": req.urlext,
-                "status_code": req.status_code,
-                "http100": req.http100,
-                "protocol": req.protocol,
-                "reason": req.reason,
-                "verb": req.method,
-                "tag": req.tag,
-                "initiator_id": req.initiator_id,
-                "group_id": req.group_id,
-                "in_progress": req.in_progress,
-                "is_server": not req.is_client,
-                "tbegin": req.tbegin.isoformat(),
-                "last_update": req.last_update.isoformat(),
-            }
-
-        for id, initiator in records.initiators.items():
-            payload["initiators"][id] = initiator.to_json(full=True)
-
-        for id, group in records.groups.items():
-            payload["groups"][id] = group.to_json()
-
-        return payload