httpdbg 1.1.1__tar.gz → 1.2.0__tar.gz

{httpdbg-1.1.1 → httpdbg-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: httpdbg
-Version: 1.1.1
+Version: 1.2.0
 Summary: A very simple tool to debug HTTP(S) client and server requests.
 Author-email: cle-b <cle@tictac.pm>
 License: Apache-2.0
@@ -114,7 +114,6 @@ Press CTRL+C to quit
 
 ![httpdbg 1.1.0 - pyhttpdbg -m flask --app demoflask run](https://github.com/cle-b/httpdbg/blob/main/ui_server.png?raw=true)
 
-This feature is new and not fully implemented. See [limitations](#limitations) for more details.
 
 ## initiators and groups
  
@@ -146,7 +145,8 @@ The initiator is the low-level socket method that receives the data. The group i
 
 | packages       | status                              | initiator | group     |
 |----------------|-------------------------------------|-----------|-----------|
-| flask          | supported                           | no        | yes       |
+| flask          | supported                           | -         | yes       |
+| fastapi        | supported                           | -         | yes       |
 
 ### supported test framework
 
@@ -154,8 +154,8 @@ The requests are grouped by tests.
 
 | packages       | status                              | initiator | group     |
 |----------------|-------------------------------------|-----------|-----------|
-| pytest         | supported                           | no        | yes       |
-| unittest       | supported                           | no        | yes       |
+| pytest         | supported                           | -         | yes       |
+| unittest       | supported                           | -         | yes       |
 
 ## configuration
 
@@ -218,7 +218,7 @@ The requests:
 
 Theoretically, if your HTTP client or server uses a standard Python socket, the HTTP requests will be recorded.
 
-Support for recording requests on the server side is new in `httpdbg 1.0.0` and is currently limited. For example, it doesn't work with `FastAPI`, which uses `uvloop`, but it does work with `Uvicorn` if the loop is `asyncio`. Work is in progress to improve this.
+Support for recording requests on the server side is new in `httpdbg 1.0.0`. In some cases, like with |FastAPI| using |uvloop|, we had to implement a new mechanism to make request recording possible. This is still a work in progress as we aim to support more frameworks.
 
 ## documentation
 

{httpdbg-1.1.1 → httpdbg-1.2.0}/README.md

@@ -87,7 +87,6 @@ Press CTRL+C to quit
 
 ![httpdbg 1.1.0 - pyhttpdbg -m flask --app demoflask run](https://github.com/cle-b/httpdbg/blob/main/ui_server.png?raw=true)
 
-This feature is new and not fully implemented. See [limitations](#limitations) for more details.
 
 ## initiators and groups
  
@@ -119,7 +118,8 @@ The initiator is the low-level socket method that receives the data. The group i
 
 | packages       | status                              | initiator | group     |
 |----------------|-------------------------------------|-----------|-----------|
-| flask          | supported                           | no        | yes       |
+| flask          | supported                           | -         | yes       |
+| fastapi        | supported                           | -         | yes       |
 
 ### supported test framework
 
@@ -127,8 +127,8 @@ The requests are grouped by tests.
 
 | packages       | status                              | initiator | group     |
 |----------------|-------------------------------------|-----------|-----------|
-| pytest         | supported                           | no        | yes       |
-| unittest       | supported                           | no        | yes       |
+| pytest         | supported                           | -         | yes       |
+| unittest       | supported                           | -         | yes       |
 
 ## configuration
 
@@ -191,7 +191,7 @@ The requests:
 
 Theoretically, if your HTTP client or server uses a standard Python socket, the HTTP requests will be recorded.
 
-Support for recording requests on the server side is new in `httpdbg 1.0.0` and is currently limited. For example, it doesn't work with `FastAPI`, which uses `uvloop`, but it does work with `Uvicorn` if the loop is `asyncio`. Work is in progress to improve this.
+Support for recording requests on the server side is new in `httpdbg 1.0.0`. In some cases, like with |FastAPI| using |uvloop|, we had to implement a new mechanism to make request recording possible. This is still a work in progress as we aim to support more frameworks.
 
 ## documentation
 

{httpdbg-1.1.1 → httpdbg-1.2.0}/httpdbg/__init__.py

@@ -3,6 +3,6 @@ from httpdbg.hooks.all import httprecord
 from httpdbg.records import HTTPRecords
 
 
-__version__ = "1.1.1"
+__version__ = "1.2.0"
 
 __all__ = ["httprecord", "HTTPRecords"]

httpdbg-1.2.0/httpdbg/env.py

@@ -0,0 +1,4 @@
+HTTPDBG_MULTIPROCESS_DIR = "HTTPDBG_MULTIPROCESS_DIR"
+
+HTTPDBG_LOG_LEVEL = "HTTPDBG_LOG_LEVEL"
+HTTPDBG_LOG_PATH = "HTTPDBG_LOG_PATH"

{httpdbg-1.1.1 → httpdbg-1.2.0}/httpdbg/hooks/all.py

@@ -7,15 +7,19 @@ from typing import Union
 
 from httpdbg.hooks.aiohttp import hook_aiohttp
 from httpdbg.hooks.external import watcher_external
+from httpdbg.hooks.fastapi import hook_fastapi
+from httpdbg.hooks.flask import hook_flask
 from httpdbg.hooks.generic import hook_generic
 from httpdbg.hooks.http import hook_http
 from httpdbg.hooks.httpx import hook_httpx
 from httpdbg.hooks.pytest import hook_pytest
 from httpdbg.hooks.requests import hook_requests
 from httpdbg.hooks.socket import hook_socket
+from httpdbg.hooks.starlette import hook_starlette
 from httpdbg.hooks.unittest import hook_unittest
 from httpdbg.hooks.urllib3 import hook_urllib3
-from httpdbg.hooks.flask import hook_flask
+from httpdbg.hooks.uvicorn import hook_uvicorn
+
 from httpdbg.records import HTTPRecords
 
 
@@ -33,12 +37,17 @@ def httprecord(
     with watcher_external(records, initiators, server):
         with hook_flask(records):
             with hook_socket(records):
-                with hook_http(records):
-                    with hook_httpx(records):
-                        with hook_requests(records):
-                            with hook_urllib3(records):
-                                with hook_aiohttp(records):
-                                    with hook_pytest(records):
-                                        with hook_unittest(records):
-                                            with hook_generic(records, initiators):
-                                                yield records
+                with hook_fastapi(records):
+                    with hook_starlette(records):
+                        with hook_uvicorn(records):
+                            with hook_http(records):
+                                with hook_httpx(records):
+                                    with hook_requests(records):
+                                        with hook_urllib3(records):
+                                            with hook_aiohttp(records):
+                                                with hook_pytest(records):
+                                                    with hook_unittest(records):
+                                                        with hook_generic(
+                                                            records, initiators
+                                                        ):
+                                                            yield records

httpdbg-1.2.0/httpdbg/hooks/fastapi.py

@@ -0,0 +1,156 @@
+# -*- coding: utf-8 -*-
+from collections.abc import Callable
+from contextlib import contextmanager
+import functools
+from functools import wraps
+from typing import Dict
+from typing import Generator
+from typing import Union
+
+import datetime
+
+from httpdbg.hooks.utils import decorate
+from httpdbg.hooks.utils import getcallargs
+from httpdbg.hooks.utils import undecorate
+from httpdbg.initiator import httpdbg_endpoint
+from httpdbg.records import HTTPRecords
+
+
+# decorate the function decorated by @app.get; @app.post etc.
+def set_hook_fastapi_endpoint(records: HTTPRecords, method: Callable):
+
+    @wraps(method)
+    def hook(*args, **kwargs):
+
+        with httpdbg_endpoint(
+            records,
+            method,
+            *args,
+            **kwargs,
+        ):
+            ret = method(*args, **kwargs)
+        return ret
+
+    return hook
+
+
+def set_hook_fastapi_apirouter_add_api_route(
+    records: HTTPRecords,
+    method: Callable,
+    already_mapped: Union[Dict[Callable, Callable], None] = None,
+):
+
+    @wraps(method)
+    def hook(*args, **kwargs):
+        callargs = getcallargs(method, *args, **kwargs)
+
+        param = "endpoint"
+
+        if param in callargs:
+            original_view_func = callargs[param]
+            if original_view_func not in already_mapped:
+                callargs[param] = decorate(
+                    records, callargs[param], set_hook_fastapi_endpoint
+                )
+                already_mapped[original_view_func] = callargs[param]
+            else:
+                callargs[param] = already_mapped[original_view_func]
+
+            args = [callargs[param] if x == original_view_func else x for x in args]
+            if param in kwargs:
+                kwargs[param] = callargs[param]
+
+        return method(*args, **kwargs)
+
+    return hook
+
+
+# decorate the "app" returned by fastapi.routing.get_request_handler to link the socketdata and the endpoint group
+def set_hook_fastapi_app(records: HTTPRecords, method: Callable):
+
+    @wraps(method)
+    async def hook(*args, **kwargs):
+
+        socketdata_id = None
+
+        callargs = getcallargs(method, *args, **kwargs)
+
+        if "request" in callargs:
+            if hasattr(callargs["request"], "__httpdbg_socketdata_id"):
+                socketdata_id = callargs["request"].__httpdbg_socketdata_id
+
+        with httpdbg_endpoint(
+            records,
+            method,
+            *args,
+            **kwargs,
+        ) as group:
+            if group and (group.id in records.groups):
+                if socketdata_id and (socketdata_id in records._sockets):
+                    records.groups[group.id].tbegin = records._sockets[
+                        socketdata_id
+                    ].record.tbegin - datetime.timedelta(milliseconds=1)
+                    records._sockets[socketdata_id].record.group_id = group.id
+
+            ret = await method(*args, **kwargs)
+
+        return ret
+
+    return hook
+
+
+def set_hook_fastapi_routing_get_request_handler(
+    records: HTTPRecords, method: Callable
+):
+
+    @wraps(method)
+    def hook(*args, **kwargs):
+
+        app = method(*args, **kwargs)
+
+        app = decorate(records, app, set_hook_fastapi_app)
+
+        return app
+
+    return hook
+
+
+@contextmanager
+def hook_fastapi(records: HTTPRecords) -> Generator[None, None, None]:
+    hooks = False
+    try:
+        import fastapi.routing
+
+        already_mapped: Dict[Callable, Callable] = {}
+
+        set_hook_fastapi_apirouter_add_api_route_with_already_mapped = (
+            functools.partial(
+                set_hook_fastapi_apirouter_add_api_route, already_mapped=already_mapped
+            )
+        )
+
+        fastapi.routing.APIRouter.add_api_route = decorate(
+            records,
+            fastapi.routing.APIRouter.add_api_route,
+            set_hook_fastapi_apirouter_add_api_route_with_already_mapped,
+        )
+
+        fastapi.routing.get_request_handler = decorate(
+            records,
+            fastapi.routing.get_request_handler,
+            set_hook_fastapi_routing_get_request_handler,
+        )
+
+        hooks = True
+    except ImportError:
+        pass
+
+    yield
+
+    if hooks:
+        fastapi.routing.APIRouter.add_api_route = undecorate(
+            fastapi.routing.APIRouter.add_api_route
+        )
+        fastapi.routing.get_request_handler = undecorate(
+            fastapi.routing.get_request_handler
+        )

{httpdbg-1.1.1 → httpdbg-1.2.0}/httpdbg/hooks/flask.py

@@ -1,8 +1,11 @@
 # -*- coding: utf-8 -*-
 from collections.abc import Callable
 from contextlib import contextmanager
+import functools
 from functools import wraps
+from typing import Dict
 from typing import Generator
+from typing import Union
 
 from httpdbg.hooks.utils import getcallargs
 from httpdbg.hooks.utils import decorate
@@ -30,12 +33,11 @@ def set_hook_flask_endpoint(records: HTTPRecords, method: Callable):
     return hook
 
 
-# we must not apply the hook more than once on a mapped endpoint function
-# AssertionError: View function mapping is overwriting an existing endpoint function: xxxx
-already_mapped = {}
-
-
-def set_hook_flask_add_url_rule(records: HTTPRecords, method: Callable):
+def set_hook_flask_add_url_rule(
+    records: HTTPRecords,
+    method: Callable,
+    already_mapped: Union[Dict[Callable, Callable], None] = None,
+):
 
     def hook(*args, **kwargs):
 
@@ -62,7 +64,11 @@ def set_hook_flask_add_url_rule(records: HTTPRecords, method: Callable):
     return hook
 
 
-def set_hook_flask_register_error_handler(records: HTTPRecords, method: Callable):
+def set_hook_flask_register_error_handler(
+    records: HTTPRecords,
+    method: Callable,
+    already_mapped: Union[Dict[Callable, Callable], None] = None,
+):
 
     def hook(*args, **kwargs):
 
@@ -95,14 +101,28 @@ def hook_flask(records: HTTPRecords) -> Generator[None, None, None]:
     try:
         import flask
 
+        # we must not apply the hook more than once on a mapped endpoint function
+        # AssertionError: View function mapping is overwriting an existing endpoint function: xxxx
+        already_mapped: Dict[Callable, Callable] = {}
+
+        set_hook_flask_add_url_rule_with_already_mapped = functools.partial(
+            set_hook_flask_add_url_rule, already_mapped=already_mapped
+        )
+
         flask.Flask.add_url_rule = decorate(
-            records, flask.Flask.add_url_rule, set_hook_flask_add_url_rule
+            records,
+            flask.Flask.add_url_rule,
+            set_hook_flask_add_url_rule_with_already_mapped,
+        )
+
+        set_hook_flask_register_error_handler_with_already_mapped = functools.partial(
+            set_hook_flask_register_error_handler, already_mapped=already_mapped
         )
 
         flask.Flask.register_error_handler = decorate(
             records,
             flask.Flask.register_error_handler,
-            set_hook_flask_register_error_handler,
+            set_hook_flask_register_error_handler_with_already_mapped,
         )
         flask.Flask.handle_exception = decorate(
             records, flask.Flask.handle_exception, set_hook_flask_endpoint

{httpdbg-1.1.1 → httpdbg-1.2.0}/httpdbg/hooks/pytest.py

@@ -12,13 +12,15 @@ from httpdbg.log import logger
 from httpdbg.records import HTTPRecords
 
 
-def set_hook_for_pytest_fixture(_: HTTPRecords, method: Callable):
+def set_hook_for_pytest_fixture(records: HTTPRecords, method: Callable):
     def hook(*args, **kwargs):
         logger().info("SET_HOOK_FOR_PYTEST_FIXTURE")
         callargs = getcallargs(method, *args, **kwargs)
 
         if "fixturefunc" in callargs:
-            with httpdbg_tag(getattr(callargs["fixturefunc"], "__name__", "fixture")):
+            with httpdbg_tag(
+                records, getattr(callargs["fixturefunc"], "__name__", "fixture")
+            ):
                 return method(*args, **kwargs)
         else:
             return method(*args, **kwargs)

{httpdbg-1.1.1 → httpdbg-1.2.0}/httpdbg/hooks/socket.py

@@ -186,46 +186,53 @@ def set_hook_for_socket_recv_into(records: HTTPRecords, method: Callable):
 
         nbytes = method(self, buffer, *args, **kwargs)
 
-        if socketdata:
-            if socketdata.record:
-                logger().info(
-                    f"RECV_INTO (after) - id={id(self)} buffer={(b''+buffer)[:20]}"
-                )
-                socketdata.record.receive_data(buffer[:nbytes])
-            else:
-                socketdata.rawdata += buffer[:nbytes]
-                http_detected = socketdata.http_detected()
-                if http_detected:
-                    logger().info("RECV_INTO - http detected")
+        if buffer:  # it appears that the buffer may be None (observed on Windows).
+            if socketdata:
+                if socketdata.record:
                     logger().info(
                         f"RECV_INTO (after) - id={id(self)} buffer={(b''+buffer)[:20]}"
                     )
-                    with httpdbg_initiator(
-                        records,
-                        traceback.extract_stack(),
-                        method,
-                        self,
-                        buffer,
-                        *args,
-                        **kwargs,
-                    ) as initiator_and_group:
-                        initiator, group, is_new = initiator_and_group
-                        if is_new:
-                            tbegin = socketdata.tbegin - datetime.timedelta(
-                                milliseconds=1
-                            )
-                            initiator.tbegin = tbegin
-                            group.tbegin = tbegin
-                        socketdata.record = HTTPRecord(
-                            tbegin=socketdata.tbegin, is_client=False
+                    socketdata.record.receive_data(buffer[:nbytes])
+                else:
+                    socketdata.rawdata += buffer[:nbytes]
+                    http_detected = socketdata.http_detected()
+                    if http_detected:
+                        logger().info("RECV_INTO - http detected")
+                        logger().info(
+                            f"RECV_INTO (after) - id={id(self)} buffer={(b''+buffer)[:20]}"
                         )
-                        socketdata.record.address = socketdata.address
-                        socketdata.record.ssl = socketdata.ssl
-                        socketdata.record.receive_data(socketdata.rawdata)
-                        if records.server:
-                            records.requests[socketdata.record.id] = socketdata.record
-                elif http_detected is False:  # if None, there is nothing to do
-                    records._sockets[id(self)] = None
+                        with httpdbg_initiator(
+                            records,
+                            traceback.extract_stack(),
+                            method,
+                            self,
+                            buffer,
+                            *args,
+                            **kwargs,
+                        ) as initiator_and_group:
+                            initiator, group, is_new = initiator_and_group
+                            if is_new:
+                                tbegin = socketdata.tbegin - datetime.timedelta(
+                                    milliseconds=1
+                                )
+                                initiator.tbegin = tbegin
+                                group.tbegin = tbegin
+                            socketdata.record = HTTPRecord(
+                                records.current_initiator,
+                                records.current_group,
+                                records.current_tag,
+                                tbegin=socketdata.tbegin,
+                                is_client=False,
+                            )
+                            socketdata.record.address = socketdata.address
+                            socketdata.record.ssl = socketdata.ssl
+                            socketdata.record.receive_data(socketdata.rawdata)
+                            if records.server:
+                                records.requests[socketdata.record.id] = (
+                                    socketdata.record
+                                )
+                    elif http_detected is False:  # if None, there is nothing to do
+                        records._sockets[id(self)] = None
 
         return nbytes
 
@@ -270,7 +277,11 @@ def set_hook_for_socket_recv(records: HTTPRecords, method: Callable):
                             initiator.tbegin = tbegin
                             group.tbegin = tbegin
                         socketdata.record = HTTPRecord(
-                            tbegin=socketdata.tbegin, is_client=False
+                            records.current_initiator,
+                            records.current_group,
+                            records.current_tag,
+                            tbegin=socketdata.tbegin,
+                            is_client=False,
                         )
                         socketdata.record.address = socketdata.address
                         socketdata.record.ssl = socketdata.ssl
@@ -320,7 +331,12 @@ def set_hook_for_socket_sendall(records: HTTPRecords, method: Callable):
                             )
                             initiator.tbegin = tbegin
                             group.tbegin = tbegin
-                        socketdata.record = HTTPRecord(tbegin=socketdata.tbegin)
+                        socketdata.record = HTTPRecord(
+                            records.current_initiator,
+                            records.current_group,
+                            records.current_tag,
+                            tbegin=socketdata.tbegin,
+                        )
                         socketdata.record.address = socketdata.address
                         socketdata.record.ssl = socketdata.ssl
                         socketdata.record.send_data(socketdata.rawdata)
@@ -371,7 +387,12 @@ def set_hook_for_socket_send(records: HTTPRecords, method: Callable):
                             )
                             initiator.tbegin = tbegin
                             group.tbegin = tbegin
-                        socketdata.record = HTTPRecord(tbegin=socketdata.tbegin)
+                        socketdata.record = HTTPRecord(
+                            records.current_initiator,
+                            records.current_group,
+                            records.current_tag,
+                            tbegin=socketdata.tbegin,
+                        )
                         socketdata.record.address = socketdata.address
                         socketdata.record.ssl = socketdata.ssl
                         socketdata.record.send_data(socketdata.rawdata)
@@ -446,7 +467,12 @@ def set_hook_for_sslobject_write(records: HTTPRecords, method: Callable):
                             )
                             initiator.tbegin = tbegin
                             group.tbegin = tbegin
-                        socketdata.record = HTTPRecord(tbegin=socketdata.tbegin)
+                        socketdata.record = HTTPRecord(
+                            records.current_initiator,
+                            records.current_group,
+                            records.current_tag,
+                            tbegin=socketdata.tbegin,
+                        )
                         socketdata.record.address = socketdata.address
                         socketdata.record.ssl = socketdata.ssl
                         socketdata.record.send_data(socketdata.rawdata)

httpdbg-1.2.0/httpdbg/hooks/starlette.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+from collections.abc import Callable
+from contextlib import contextmanager
+from functools import wraps
+from typing import Generator
+
+from httpdbg.hooks.utils import decorate
+from httpdbg.hooks.utils import getcallargs
+from httpdbg.hooks.utils import undecorate
+from httpdbg.records import HTTPRecords
+
+
+def set_hook_starlette_request_init(records: HTTPRecords, method: Callable):
+
+    @wraps(method)
+    def hook(self, *args, **kwargs):
+
+        callargs = getcallargs(method, self, *args, **kwargs)
+
+        if "receive" in callargs:
+            if hasattr(callargs["receive"], "__self__"):
+                if hasattr(callargs["receive"].__self__, "transport"):
+                    if hasattr(
+                        callargs["receive"].__self__.transport,
+                        "_TCPTransport__httpdbg_socketdata",
+                    ):
+                        # make the link between starlette.requests.Request (self) and uvicorn.protocols.http.httptools_impl.HttpToolsProtocol (socketdata)
+                        self.__httpdbg_socketdata_id = callargs[
+                            "receive"
+                        ].__self__.transport._TCPTransport__httpdbg_socketdata.id
+
+        return method(self, *args, **kwargs)
+
+    return hook
+
+
+@contextmanager
+def hook_starlette(records: HTTPRecords) -> Generator[None, None, None]:
+    hooks = False
+    try:
+        import starlette.requests
+
+        starlette.requests.Request.__init__ = decorate(
+            records,
+            starlette.requests.Request.__init__,
+            set_hook_starlette_request_init,
+        )
+
+        hooks = True
+    except ImportError:
+        pass
+
+    yield
+
+    if hooks:
+        starlette.requests.Request.__init__ = undecorate(
+            starlette.requests.Request.__init__
+        )

{httpdbg-1.1.1 → httpdbg-1.2.0}/httpdbg/hooks/unittest.py

@@ -45,15 +45,15 @@ def set_hook_for_unittest_run(records: HTTPRecords, method: Callable):
     return hook
 
 
-def set_hook_for_unittest_fixture(_: HTTPRecords, method: Callable):
+def set_hook_for_unittest_fixture(records: HTTPRecords, method: Callable):
 
     def hook(*args, **kwargs):
 
         if getattr(method, "__name__") == "_callSetUp":
-            with httpdbg_tag("setUp"):
+            with httpdbg_tag(records, "setUp"):
                 return method(*args, **kwargs)
         elif getattr(method, "__name__") == "_callTearDown":
-            with httpdbg_tag("tearDown"):
+            with httpdbg_tag(records, "tearDown"):
                 return method(*args, **kwargs)
         else:
             return method(*args, **kwargs)