httpdbg 0.31.6__tar.gz → 0.32.0__tar.gz

{httpdbg-0.31.6 → httpdbg-0.32.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: httpdbg
-Version: 0.31.6
+Version: 0.32.0
 Summary: A very simple tool to debug HTTP(S) client requests
 Author-email: cle-b <cle@tictac.pm>
 License: Apache-2.0

{httpdbg-0.31.6 → httpdbg-0.32.0}/httpdbg/__init__.py

@@ -3,6 +3,6 @@ from httpdbg.hooks.all import httprecord
 from httpdbg.records import HTTPRecords
 
 
-__version__ = "0.31.6"
+__version__ = "0.32.0"
 
 __all__ = ["httprecord", "HTTPRecords"]

{httpdbg-0.31.6 → httpdbg-0.32.0}/httpdbg/env.py

@@ -1,7 +1,7 @@
 HTTPDBG_CURRENT_GROUP = "HTTPDBG_CURRENT_GROUP"
 HTTPDBG_CURRENT_INITIATOR = "HTTPDBG_CURRENT_INITIATOR"
 HTTPDBG_CURRENT_TAG = "HTTPDBG_CURRENT_TAG"
-HTTPDBG_SUBPROCESS_DIR = "HTTPDBG_SUBPROCESS_DIR"
+HTTPDBG_MULTIPROCESS_DIR = "HTTPDBG_MULTIPROCESS_DIR"
 
 HTTPDBG_LOG_LEVEL = "HTTPDBG_LOG_LEVEL"
 HTTPDBG_LOG_PATH = "HTTPDBG_LOG_PATH"

httpdbg-0.32.0/httpdbg/hooks/external.py

@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+from contextlib import contextmanager
+from typing import Generator
+import glob
+import os
+import pickle
+import shutil
+import tempfile
+import time
+import threading
+
+from httpdbg.env import HTTPDBG_MULTIPROCESS_DIR
+from httpdbg.log import logger
+from httpdbg.records import HTTPRecords
+
+
+@contextmanager
+def watcher_external(records: HTTPRecords) -> Generator[HTTPRecords, None, None]:
+    if HTTPDBG_MULTIPROCESS_DIR not in os.environ:
+        with tempfile.TemporaryDirectory(prefix="httpdbg_") as httpdbg_multiprocess_dir:
+
+            logger().info(f"watcher_external {httpdbg_multiprocess_dir}")
+            os.environ[HTTPDBG_MULTIPROCESS_DIR] = httpdbg_multiprocess_dir
+
+            # we use a custom sitecustomize.py script to record the request in the subprocesses.
+            # It doesn't work if the subprocess is created using the "fork" method
+            # instead of the "spawn" method.
+            template = os.path.join(
+                os.path.dirname(os.path.abspath(__file__)),
+                "template_sitecustomize.py",
+            )
+            sitecustomize = os.path.join(httpdbg_multiprocess_dir, "sitecustomize.py")
+
+            shutil.copy(template, sitecustomize)
+
+            if "PYTHONPATH" in os.environ:
+                os.environ["PYTHONPATH"] = (
+                    f"{httpdbg_multiprocess_dir}:{os.environ['PYTHONPATH']}"
+                )
+            else:
+                os.environ["PYTHONPATH"] = httpdbg_multiprocess_dir
+
+            try:
+                watcher = WatcherSubprocessDirThread(
+                    records, httpdbg_multiprocess_dir, 1.0
+                )
+                watcher.start()
+
+                yield records
+            finally:
+                watcher.shutdown()
+    else:
+        yield records
+
+
+class WatcherSubprocessDirThread(threading.Thread):
+    def __init__(self, records: HTTPRecords, directory: str, delay: float) -> None:
+        self.records: HTTPRecords = records
+        self.directory: str = directory
+        self.delay: float = delay
+        self._running: bool = True
+        threading.Thread.__init__(self)
+
+    def load_dump(self):
+        for dump in glob.glob(os.path.join(self.directory, "*.httpdbgrecords")):
+            with open(dump, "rb") as dumpfile:
+                newrecords: HTTPRecords = pickle.load(dumpfile)
+                self.records.requests.update(newrecords.requests)
+                self.records.initiators.update(newrecords.initiators)
+                self.records.groups.update(newrecords.groups)
+
+    def run(self):
+        while self._running:
+            self.load_dump()
+            time.sleep(self.delay)
+
+    def shutdown(self):
+        self._running = False
+        self.join(timeout=5)
+        self.load_dump()

httpdbg-0.32.0/httpdbg/hooks/template_sitecustomize.py

@@ -0,0 +1,65 @@
+import atexit
+import os
+import pickle
+import sys
+import time
+import threading
+from typing import Union
+import uuid
+
+from httpdbg.hooks.all import httprecord
+from httpdbg import HTTPRecords
+
+
+class HttpdbgRecorder:
+    """Records the HTTP requests in subprocesses and saves them to disk."""
+
+    def __init__(self):
+        self._running: bool = False
+        self.fname: str = os.path.join(
+            os.environ["HTTPDBG_MULTIPROCESS_DIR"], str(uuid.uuid1())
+        )
+        self.context = None
+        self.records: Union[HTTPRecords, None] = None
+
+    def _save_to_disk_loop(self):
+        while self._running:
+            self.save_to_disk()
+            time.sleep(1)
+
+    def save_to_disk(self):
+        if self.records and len(self.records.requests) > 0:
+            with open(f"{self.fname}.httpdbgrecords.tmp", "wb") as f:
+                pickle.dump(self.records, f)
+            os.replace(
+                f"{self.fname}.httpdbgrecords.tmp", f"{self.fname}.httpdbgrecords"
+            )
+
+    def start(self):
+        """Start recording."""
+        self.context = httprecord()
+        self.records = self.context.__enter__()
+        self._running = True
+        self._thread = threading.Thread(target=self._save_to_disk_loop, daemon=True)
+        self._thread.start()
+
+    def stop(self):
+        """Stop recording."""
+        self._running = False
+        if self._thread:
+            self._thread.join(timeout=5)
+        self.save_to_disk()
+        if self.context:
+            self.context.__exit__(None, None, None)
+
+
+if "HTTPDBG_MULTIPROCESS_DIR" in os.environ:
+
+    if not hasattr(sys, "_httpdbg_enabled"):
+        sys._httpdbg_enabled = True  # type: ignore[attr-defined]
+
+        # start recording when the process starts and stop it when process exists
+        httprecorder = HttpdbgRecorder()
+        httprecorder.start()
+
+        atexit.register(httprecorder.stop)

{httpdbg-0.31.6 → httpdbg-0.32.0}/httpdbg/mode_script.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
-import importlib
+import os
+import runpy
 import sys
 import traceback
 from typing import List
@@ -8,24 +9,25 @@ from httpdbg.log import logger
 
 
 def run_script(argv: List[str]) -> None:
-    sys.argv = argv
+
     if len(argv) == 0:
         exit("script mode - error - python file required, but none set")
+
     try:
-        spec = importlib.util.spec_from_file_location("__main__", argv[0])  # type: ignore
-        module = importlib.util.module_from_spec(spec)  # type: ignore
-    except AttributeError:
-        exit("script mode - error - the first argument shall be a python file")
-    try:
-        original__main___module = sys.modules["__main__"]
-        sys.modules["__main__"] = module  # mandatory to execute a unittest test file.
-        spec.loader.exec_module(module)  # type: ignore
-    except Exception:
-        traceback.print_exc()
+        original_sys_path = sys.path
+        original_sys_argv = sys.argv
+
+        sys.path.insert(0, os.path.abspath(argv[0]))
+        sys.argv = argv
+
+        runpy.run_path(argv[0], run_name="__main__")
     except SystemExit as ex:
         # in some case, for example when executing unittest tests from a script,
         # the SystemExit exception is raised by the script itself so we catch it.
         # TODO propagate the SystemExit value when httpdbg terminates.
         logger().info(f"run_script - exception SystemExit - {str(ex)}")
+    except Exception:
+        traceback.print_exc()
     finally:
-        sys.modules["__main__"] = original__main___module
+        sys.path = original_sys_path
+        sys.argv = original_sys_argv

{httpdbg-0.31.6 → httpdbg-0.32.0}/httpdbg.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: httpdbg
-Version: 0.31.6
+Version: 0.32.0
 Summary: A very simple tool to debug HTTP(S) client requests
 Author-email: cle-b <cle@tictac.pm>
 License: Apache-2.0

{httpdbg-0.31.6 → httpdbg-0.32.0}/httpdbg.egg-info/SOURCES.txt

@@ -31,6 +31,7 @@ httpdbg/hooks/httpx.py
 httpdbg/hooks/pytest.py
 httpdbg/hooks/requests.py
 httpdbg/hooks/socket.py
+httpdbg/hooks/template_sitecustomize.py
 httpdbg/hooks/unittest.py
 httpdbg/hooks/urllib3.py
 httpdbg/hooks/utils.py
@@ -78,5 +79,6 @@ tests/test_initiator.py
 tests/test_mode_console.py
 tests/test_mode_module_pytest.py
 tests/test_mode_script.py
+tests/test_multiprocess.py
 tests/test_preview.py
 tests/test_server.py

{httpdbg-0.31.6 → httpdbg-0.32.0}/tests/test_mode_script.py

@@ -67,8 +67,8 @@ def test_run_script_no_args(httpbin, capsys):
 
 @pytest.mark.script
 def test_run_script_not_a_python_script(httpbin, capsys):
-    with pytest.raises(SystemExit):
-        run_script(["not_a_path_to_a_python_script"])
+    # should not raise an exception
+    run_script(["not_a_path_to_a_python_script"])
 
 
 @pytest.mark.api

httpdbg-0.32.0/tests/test_multiprocess.py

@@ -0,0 +1,52 @@
+import os
+import platform
+
+import pytest
+
+from httpdbg import httprecord
+from httpdbg.env import HTTPDBG_MULTIPROCESS_DIR
+from httpdbg.mode_script import run_script
+
+
+@pytest.mark.script
+def test_run_script_multithread(httpbin):
+
+    if HTTPDBG_MULTIPROCESS_DIR in os.environ:
+        del os.environ[HTTPDBG_MULTIPROCESS_DIR]
+
+    with httprecord() as records:
+        os.environ["HTTPDBG_TEST_MULTITHREAD_BASE_URL"] = httpbin.url
+        script_to_run = os.path.join(
+            os.path.dirname(os.path.realpath(__file__)), "demo_run_multithread.py"
+        )
+        run_script([script_to_run, httpbin.url])
+
+    assert len(records) == 3
+    pids = []
+    for record in records:
+        pids.append(record.url.split("/")[-1])
+    assert len(set(pids)) == 1
+
+
+@pytest.mark.script
+@pytest.mark.xfail(
+    platform.system().lower() == "windows",
+    reason="the test fails on the CI but the feature works on windows",
+)
+def test_run_script_multiprocess(httpbin):
+
+    if HTTPDBG_MULTIPROCESS_DIR in os.environ:
+        del os.environ[HTTPDBG_MULTIPROCESS_DIR]
+
+    with httprecord() as records:
+        os.environ["HTTPDBG_TEST_MULTIPROCESS_BASE_URL"] = httpbin.url
+        script_to_run = os.path.join(
+            os.path.dirname(os.path.realpath(__file__)), "demo_run_multiprocess.py"
+        )
+        run_script([script_to_run, httpbin.url])
+
+    assert len(records) == 3
+    pids = []
+    for record in records:
+        pids.append(record.url.split("/")[-1])
+    assert len(set(pids)) == 3

httpdbg-0.31.6/httpdbg/hooks/external.py

@@ -1,61 +0,0 @@
-# -*- coding: utf-8 -*-
-from contextlib import contextmanager
-from typing import Generator
-import glob
-import os
-import pickle
-import tempfile
-import time
-import threading
-
-from httpdbg.env import HTTPDBG_SUBPROCESS_DIR
-from httpdbg.records import HTTPRecords
-
-
-@contextmanager
-def watcher_external(records: HTTPRecords) -> Generator[HTTPRecords, None, None]:
-    try:
-        watcher = None
-        if HTTPDBG_SUBPROCESS_DIR not in os.environ:
-            with tempfile.TemporaryDirectory(
-                prefix="httpdbg"
-            ) as httpdbg_subprocess_dir:
-                os.environ[HTTPDBG_SUBPROCESS_DIR] = httpdbg_subprocess_dir
-
-                watcher = WatcherSubprocessDirThread(records)
-                watcher.start()
-
-                yield records
-
-                watcher.shutdown()
-                watcher.join()
-        else:
-            yield records
-    except Exception as ex:
-        if watcher:
-            watcher.shutdown()
-        raise ex
-
-
-class WatcherSubprocessDirThread(threading.Thread):
-    def __init__(self, records: HTTPRecords, delay: int = 2) -> None:
-        self.stop = False
-        self.records = records
-        self.delay = delay
-        threading.Thread.__init__(self)
-
-    def load_dump(self):
-        for dump in glob.glob(f"{os.environ[HTTPDBG_SUBPROCESS_DIR]}/*.httpdbgrecords"):
-            with open(dump, "rb") as dumpfile:
-                newrecords: HTTPRecords = pickle.load(dumpfile)
-                self.records.requests.update(newrecords.requests)
-                self.records.initiators.update(newrecords.initiators)
-                self.records.groups.update(newrecords.groups)
-
-    def run(self):
-        while not self.stop:
-            time.sleep(self.delay)
-            self.load_dump()
-
-    def shutdown(self):
-        self.stop = True