httpdbg 2.0.1__tar.gz → 2.1.0__tar.gz

{httpdbg-2.0.1 → httpdbg-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: httpdbg
-Version: 2.0.1
+Version: 2.1.0
 Summary: A very simple tool to debug HTTP(S) client and server requests.
 Author-email: cle-b <cle@tictac.pm>
 License: Apache-2.0
@@ -192,6 +192,8 @@ options:
   --only-client         record only HTTP client requests
   --keep-up, -k         keep the server up even if the requests have been read
   --force-quit, -q      stop the server even if the requests have not been read
+  --export-html EXPORT_HTML
+                        the path to the export file
   --console             run a python console (default)
   --module MODULE, -m MODULE
                         run library module as a script (the next args are passed to pytest as is)
@@ -225,6 +227,10 @@ The requests:
  * are still available in the web page even if the python process stopped (except if you force quit before the requests have been loaded by the web page).
  * are automatically cleaned if a new execution is detected.
 
+## export
+
+You can export the HTTP traces in a single HTML file. In that case, the web interface is not available during the execution of the python command. 
+
 ## limitations
 
 Theoretically, if your HTTP client or server uses a standard Python socket, the HTTP requests will be recorded.

{httpdbg-2.0.1 → httpdbg-2.1.0}/README.md

@@ -166,6 +166,8 @@ options:
   --only-client         record only HTTP client requests
   --keep-up, -k         keep the server up even if the requests have been read
   --force-quit, -q      stop the server even if the requests have not been read
+  --export-html EXPORT_HTML
+                        the path to the export file
   --console             run a python console (default)
   --module MODULE, -m MODULE
                         run library module as a script (the next args are passed to pytest as is)
@@ -199,6 +201,10 @@ The requests:
  * are still available in the web page even if the python process stopped (except if you force quit before the requests have been loaded by the web page).
  * are automatically cleaned if a new execution is detected.
 
+## export
+
+You can export the HTTP traces in a single HTML file. In that case, the web interface is not available during the execution of the python command. 
+
 ## limitations
 
 Theoretically, if your HTTP client or server uses a standard Python socket, the HTTP requests will be recorded.

httpdbg-2.1.0/httpdbg/__init__.py

@@ -0,0 +1,20 @@
+__version__ = "2.1.0"
+
+__all__ = ["export_html", "httprecord", "HTTPRecords"]
+
+
+# lazy loading to avoid circular import issue
+def __getattr__(name):
+    if name == "export_html":
+        from httpdbg.export import export_html
+
+        return export_html
+    if name == "httprecord":
+        from httpdbg.hooks.all import httprecord
+
+        return httprecord
+    if name == "HTTPRecords":
+        from httpdbg.records import HTTPRecords
+
+        return HTTPRecords
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

httpdbg-2.1.0/httpdbg/__main__.py

@@ -0,0 +1,93 @@
+import argparse
+from pathlib import Path
+import sys
+import time
+from typing import Union
+
+from httpdbg import __version__
+from httpdbg.args import read_args
+from httpdbg.export import export_html
+from httpdbg.hooks.all import httprecord
+from httpdbg.log import set_env_for_logging
+from httpdbg.server import httpdbg_srv
+from httpdbg.mode_console import run_console
+from httpdbg.mode_module import run_module
+from httpdbg.mode_script import run_script
+from httpdbg.records import HTTPRecords
+
+
+def print_msg(msg):
+    sep = ".... - - .--. -.. -... --. .... - - .--. -.. -... --. .... - - .--. -.. -... --."
+    msg = f"{sep}\n{msg}\n{sep}"
+    print(msg)
+
+
+def pyhttpdbg(
+    params: argparse.Namespace, subparams: list[str], test_mode: bool = False
+):
+
+    set_env_for_logging(params.log_level, params.log)
+
+    url = f"http://{params.host}:{params.port}/{'?hi=on' if params.console else ''}"
+
+    if not params.no_banner:
+        print_msg(f"  httpdbg - HTTP(S) requests available at {url}")
+
+    sys.path.insert(0, "")  # to mimic the default python command behavior
+
+    def run_recorder(
+        records: HTTPRecords,
+        initiators: Union[list[str], None],
+        record_server: bool,
+        test_mode: bool,
+    ):
+        records.server = record_server
+        with httprecord(records, initiators, server=records.server):
+            if params.module:
+                run_module(subparams)
+            elif params.script:
+                run_script(subparams)
+            else:
+                run_console(records, test_mode)
+
+    if params.export_html:
+        records = HTTPRecords()
+
+        run_recorder(records, params.initiator, not params.only_client, test_mode)
+
+        export_html(records, Path(params.export_html))
+
+    else:
+        with httpdbg_srv(params.host, params.port) as records:
+
+            run_recorder(records, params.initiator, not params.only_client, test_mode)
+
+            if not (params.force_quit or test_mode):
+                if not params.no_banner:
+                    print_msg(f"  httpdbg - HTTP(S) requests available at {url}")
+
+                if params.keep_up:
+                    input("Press enter to quit")
+                else:
+                    # we keep the server up until all the requests have been loaded in the web interface
+                    print(
+                        "Waiting until all the requests have been loaded in the web interface."
+                    )
+                    print("Press Ctrl+C to quit.")
+                    try:
+                        while records.unread:
+                            time.sleep(0.5)
+                    except KeyboardInterrupt:  # pragma: no cover
+                        pass
+
+
+def pyhttpdbg_entry_point(test_mode=False):
+    params, subparams = read_args(sys.argv[1:])
+    if params.version:
+        print(__version__)
+    else:
+        pyhttpdbg(params, subparams, test_mode=test_mode)
+
+
+if __name__ == "__main__":
+    pyhttpdbg_entry_point()

{httpdbg-2.0.1 → httpdbg-2.1.0}/httpdbg/args.py

@@ -40,22 +40,28 @@ def read_args(args: list[str]) -> tuple[argparse.Namespace, list[str]]:
         help="record only HTTP client requests",
     )
 
-    server_state = parser.add_mutually_exclusive_group()
+    server_or_export = parser.add_mutually_exclusive_group()
 
-    server_state.add_argument(
+    server_or_export.add_argument(
         "--keep-up",
         "-k",
         action="store_true",
         help="keep the server up even if the requests have been read",
     )
 
-    server_state.add_argument(
+    server_or_export.add_argument(
         "--force-quit",
         "-q",
         action="store_true",
         help="stop the server even if the requests have not been read",
     )
 
+    server_or_export.add_argument(
+        "--export-html",
+        type=str,
+        help="the path to the export file",
+    )
+
     parser.add_argument(
         "--log-level",
         type=LogLevel,

httpdbg-2.1.0/httpdbg/export.py

@@ -0,0 +1,109 @@
+import base64
+import json
+from pathlib import Path
+import re
+import xml.etree.ElementTree as ET
+
+import httpdbg
+from httpdbg import HTTPRecords
+from httpdbg.webapp.api import RequestListPayload, RequestPayload
+
+
+def generate_html(records: HTTPRecords, for_export: bool = True) -> str:
+
+    current_dir = Path(__file__).resolve().parent
+
+    with open(Path(current_dir) / "webapp/static/index.htm") as findexhtml:
+        html = findexhtml.read()
+
+    html = html.replace("$**HTTPDBG_VERSION**$", httpdbg.__version__)
+
+    # favicon
+    with open(current_dir / "webapp/static/favicon.ico", "rb") as ffavicon:
+        b64_icon = base64.b64encode(ffavicon.read()).decode("utf-8")
+        data_uri = f"data:image/x-icon;base64,{b64_icon}"
+        html = html.replace(
+            '<link rel="shortcut icon" href="static/favicon.ico">',
+            f'<link rel="icon" type="image/x-icon" href="{data_uri}">',
+        )
+
+    # icons
+    def svg_file_to_symbol(svg_path: Path, symbol_id: str) -> str:
+        ET.register_namespace("", "http://www.w3.org/2000/svg")
+        root = ET.parse(svg_path).getroot()
+        symbol = ET.Element(
+            "symbol", {"id": symbol_id, "viewBox": root.attrib["viewBox"]}
+        )
+        for child in list(root):
+            symbol.append(child)
+        return ET.tostring(symbol, encoding="unicode")
+
+    icons_inline = ""
+    for icon_path in (current_dir / "webapp/static/icons").glob("*.svg"):
+        icons_inline += "\n        " + svg_file_to_symbol(
+            icon_path, icon_path.name[:-4]
+        )
+    icons_inline = f'<svg width="0" height="0" style="position:absolute;visibility:hidden" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">{icons_inline}\n    </svg>'
+    html = html.replace("$**PRELOAD_ICONS**$", icons_inline)
+
+    # css
+    stylesheet_pattern = (
+        r'<link\s+[^>]*rel=["\']stylesheet["\'][^>]*href=["\']([^"\']+)["\']'
+    )
+    for stylesheet in re.findall(stylesheet_pattern, html):
+        with open(Path(current_dir) / "webapp" / stylesheet) as fcss:
+            html = html.replace(
+                f'<link rel="stylesheet" href="{stylesheet}">',
+                f"<style>{fcss.read()}</style>",
+            )
+
+    # js
+    javascript_pattern = r'<script\s+[^>]*src=["\']([^"\']+)["\']'
+    for js in re.findall(javascript_pattern, html):
+        with open(Path(current_dir) / "webapp" / js) as fjs:
+            html = html.replace(
+                f'<script src="{js}"></script>', f"<script>{fjs.read()}</script>"
+            )
+
+    # static export of the requests data
+    if for_export:
+        static_all_requests = json.dumps(
+            records, cls=RequestListPayload, ensure_ascii=False
+        )
+        map_requests: dict[str, object] = dict()
+        for record in records:
+            map_requests[record.id] = json.loads(json.dumps(record, cls=RequestPayload))
+        static_requests: str = json.dumps(map_requests, ensure_ascii=False)
+
+        def safe_for_script_tag(s: str) -> str:
+            return s.replace("</script>", "<\\/script>")
+
+        html_export = f"""
+    <script id="all-requests" type="application/json">
+        {safe_for_script_tag(static_all_requests)}
+    </script>
+
+    <script id="requests-map" type="application/json">
+        {safe_for_script_tag(static_requests)}
+    </script>
+
+    <script>
+    global.static_all_requests = JSON.parse(
+        document.getElementById('all-requests').textContent
+    );
+    global.static_requests = JSON.parse(
+        document.getElementById('requests-map').textContent
+    );
+    </script>
+"""
+
+        html = html.replace("$**EXPORT**$", html_export)
+    else:
+        html = html.replace("$**EXPORT**$", "")
+
+    return html
+
+
+def export_html(records: HTTPRecords, filename: Path):
+    with open(filename, "w") as f:
+        f.write(generate_html(records))

{httpdbg-2.0.1 → httpdbg-2.1.0}/httpdbg/initiator.py

@@ -2,7 +2,9 @@
 from collections.abc import Callable
 from contextlib import contextmanager
 import datetime
+import functools
 import inspect
+import linecache
 import os
 import platform
 import traceback
@@ -98,9 +100,9 @@ def in_lib(line: str, packages: list[str] = None):
 def get_current_instruction(
     extracted_stack: traceback.StackSummary,
 ) -> tuple[str, str, list[str]]:
-    instruction = ""
-    short_stack = ""
-    stack = []
+    instruction: str = ""
+    short_stack: str = ""
+    stack: list[str] = []
 
     try:
         n_stack = -2
@@ -114,12 +116,7 @@ def get_current_instruction(
             n_stack -= 1
             framesummary = extracted_stack[n_stack - 1]
 
-        short_stack = f'File "{framesummary.filename}", line {framesummary.lineno}, in {framesummary.name}\n'
         if os.path.exists(framesummary.filename) and framesummary.lineno is not None:
-            instruction, s_stack = extract_short_stack_from_file(
-                framesummary.filename, framesummary.lineno, 0, 8
-            )
-            short_stack += s_stack
 
             # stack
             to_include = False
@@ -131,14 +128,20 @@ def get_current_instruction(
                     and ("importlib" not in fs.filename)
                 )  # remove the stack before to start the user part
                 if to_include:
-                    _, s_stack = extract_short_stack_from_file(
+                    instruction, s_stack, l_stack = extract_short_stack_from_file(
                         fs.filename,
                         fs.lineno if fs.lineno else 0,
-                        4 if last_stack else 0,
-                        8,
                         not last_stack,
                     )
-                    stack.append(f'File "{fs.filename}", line {fs.lineno}, \n{s_stack}')
+                    if last_stack:
+                        short_stack = (
+                            f'File "{fs.filename}", line {fs.lineno}, in {fs.name}\n'
+                        )
+                        short_stack += s_stack
+
+                    stack.append(
+                        f'File "{fs.filename}", line {fs.lineno}, \n{l_stack}\n'
+                    )
 
         else:
             instruction = framesummary.line if framesummary.line else "console"
@@ -152,52 +155,57 @@ def get_current_instruction(
     return instruction.replace("\n", " "), short_stack, stack
 
 
+@functools.cache
 def extract_short_stack_from_file(
     filename: str,
     lineno: int,
-    before: int,
-    after: int,
     stop_if_instruction_ends: bool = True,
-) -> tuple[str, str]:
-    instruction = ""
-    short_stack = ""
+) -> tuple[str, str, str]:
+    before: int = 4
+    after: int = 8
+    instruction: str = ""
+    short_stack: str = ""
+    long_stack: str = ""
 
     try:
         if os.path.exists(filename):
-            with open(filename, "r") as src:
-                lines = src.read().splitlines()
-
-                # copy the lines before
-                for i in range(
-                    max(0, lineno - 1 - before), min(lineno - 1, len(lines))
-                ):
-                    line = lines[i]
-                    short_stack += f" {i+1}. {line}\n"
-
-                # try to recompose the instruction if on multi-lines
-                end_of_instruction_found = False
-                for i in range(max(0, lineno - 1), min(lineno - 1 + after, len(lines))):
-                    line = lines[i]
-                    if not end_of_instruction_found:
-                        instruction += line.strip()
-                    short_stack += f" {i+1}. {line}{' <====' if (before > 0 and i + 1 == lineno) else ''}\n"
-                    nb_parenthesis = 0
-                    for c in instruction[instruction.find("(") :]:
-                        if c == "(":
-                            nb_parenthesis += 1
-                        if c == ")":
-                            nb_parenthesis -= 1
-                        if nb_parenthesis == 0:
-                            end_of_instruction_found = True
-                            break
-                    if end_of_instruction_found and stop_if_instruction_ends:
+            # copy the lines before, only for the "long" stack (if they are not empty)
+            copyit = False
+            for i in range(max(0, lineno - before), lineno):
+                line = linecache.getline(filename, i).removesuffix("\n").rstrip()
+                copyit = copyit or (line != "")
+                if copyit:
+                    long_stack += f" {i}. {line}\n"
+
+            # try to recompose the instruction if on multi-lines
+            end_of_instruction_found = False
+            for i in range(max(0, lineno), lineno + after):
+                line = linecache.getline(filename, i).removesuffix("\n").rstrip()
+                if not end_of_instruction_found:
+                    instruction += line.strip()
+                    short_stack += f" {i}. {line}\n"
+                long_stack += (
+                    f" {i}. {line}{' <====' if (before > 0 and i == lineno) else ''}\n"
+                )
+                nb_parenthesis = 0
+                for c in instruction[instruction.find("(") :]:
+                    if c == "(":
+                        nb_parenthesis += 1
+                    if c == ")":
+                        nb_parenthesis -= 1
+                    if nb_parenthesis == 0:
+                        end_of_instruction_found = True
                         break
+                if end_of_instruction_found and stop_if_instruction_ends:
+                    break
     except Exception as ex:
+        tb = traceback.extract_tb(ex.__traceback__)
+        _, lineno, _, _ = tb[-1]
         logger().info(
-            f"EXTRACT_SHORT_STACK_FROM_FILE {filename} lineno={lineno} before={before} after={after}- error - {str(ex)}"
+            f"EXTRACT_SHORT_STACK_FROM_FILE {filename} lineno={lineno} before={before} after={after}- error (line {lineno}) - {str(ex)}",
         )
 
-    return instruction, short_stack
+    return instruction, short_stack, long_stack
 
 
 def construct_call_str(original_method, *args, **kwargs):
@@ -345,7 +353,7 @@ def httpdbg_endpoint(
     filename = inspect.getsourcefile(original_method)
     if filename:
         _, lineno = inspect.getsourcelines(original_method)
-        instruction, s_stack = extract_short_stack_from_file(filename, lineno, 0, 8)
+        instruction, s_stack, _ = extract_short_stack_from_file(filename, lineno)
         full_label = (
             f'File "{filename}", line {lineno}, in {original_method.__name__}\n'
         )

{httpdbg-2.0.1 → httpdbg-2.1.0}/httpdbg/webapp/app.py

@@ -1,18 +1,13 @@
-# -*- coding: utf-8 -*-
 from collections.abc import Callable
 from contextlib import contextmanager
-import glob
 from http.server import BaseHTTPRequestHandler
 import json
-import mimetypes
-import os
+from pathlib import Path
 import re
 from urllib.parse import ParseResult
 from urllib.parse import parse_qs
 from urllib.parse import urlparse
 
-
-from httpdbg import __version__
 from httpdbg.log import logger
 from httpdbg.webapp.api import RequestListPayload, RequestPayload
 from httpdbg.records import HTTPRecords
@@ -56,61 +51,34 @@ class HttpbgHTTPRequestHandler(BaseHTTPRequestHandler):
                 break
 
     def serve_static(self, url: ParseResult):
-        base_path = os.path.dirname(os.path.realpath(__file__))
-
-        if not (
-            (url.path.lower() in ["/", "index.htm", "index.html"])
-            or url.path.startswith("/static/")
-        ):
-            return False
 
         if url.path.lower() in ["/", "index.htm", "index.html"]:
-            self.path = "/static/index.htm"
-
-        # get real path
-        self.path = self.path.split("-+-")[0]
-
-        fullpath = os.path.realpath(os.path.join(base_path, self.path[1:]))
+            from httpdbg.export import generate_html
 
-        if not fullpath.startswith(base_path):
-            # if the file is not in the static directory, we don't serve it
-            return self.serve_not_found(url)
-
-        if not os.path.exists(fullpath):
-            return self.serve_not_found(url)
-        else:
             self.send_response(200)
-            self.send_header(
-                "content-type", mimetypes.types_map[os.path.splitext(fullpath)[1]]
-            )
-            if self.path == "/static/index.htm":
-                self.send_header_no_cache()
-            else:
-                self.send_header_with_cache(604800)
+            self.send_header("content-type", "text/html")
+            self.send_header_no_cache()
             self.end_headers()
+            self.wfile.write(
+                generate_html(self.records, for_export=False).encode("utf-8")
+            )
+            return True
 
-            with open(fullpath, "rb") as f:
-                filecontent = f.read()
-
-                if b"$**PRELOAD_ICONS**$" in filecontent:
-                    icons = ""
-                    for icon in glob.glob(
-                        os.path.realpath(base_path) + "/static/icons/*.svg"
-                    ):
-                        icon_path = icon.replace(os.path.realpath(base_path) + "/", "")
-                        icons += f"    <link rel='preload' href='{icon_path}-+-$**HTTPDBG_VERSION**$' as='image' type='image/svg+xml' />\n"
+        if url.path.lower() == "favicon.ico":
+            self.send_response(200)
+            self.send_header("content-type", "image/x-icon")
+            self.send_header_no_cache()
+            self.end_headers()
 
-                    filecontent = filecontent.replace(
-                        b"$**PRELOAD_ICONS**$", icons.encode("utf-8")
-                    )
+            current_dir = Path(__file__).resolve().parent
 
-                filecontent = filecontent.replace(
-                    b"$**HTTPDBG_VERSION**$", __version__.encode("utf-8")
-                )
+            with open(Path(current_dir) / "static/favicon.ico") as f:
+                filecontent = f.read()
+                self.wfile.write(filecontent.encode("utf-8"))
 
-                self.wfile.write(filecontent)
+            return True
 
-        return True
+        return False
 
     def serve_requests(self, url: ParseResult):
         if not (url.path.lower() == "/requests"):

{httpdbg-2.0.1 → httpdbg-2.1.0}/httpdbg/webapp/static/api.js

@@ -5,7 +5,7 @@ const global = {
     sessions: {},
     requests: {},
     initiators: {},
-    groups: {},    
+    groups: {},
     connected: false,
     group_collapse: [],
     groupby: "default"
@@ -24,32 +24,35 @@ function save_request(request_id, request, session_id) {
         }
         request.session_id = session_id;
         request.initiator = initiator;
-    
+
         if (request.in_progress) {
-            request.status_code_view = '<img class="icon" src="static/icons/wait-sandclock-icon.svg-+-$**HTTPDBG_VERSION**$" alt="loading"/>';
+            request.status_code_view = '<svg class="icon" alt="loading"><use href="#wait-sandclock-icon"></use></svg>';
         } else {
             switch (request.status_code) {
                 case 0:
-                    request.status_code_view = '<img class="icon" src="static/icons/wait-sandclock-icon.svg-+-$**HTTPDBG_VERSION**$/" alt="loading"/>';
+                    request.status_code_view = '<svg class="icon" alt="loading"><use href="#wait-sandclock-icon"></use></svg>';
                     break;
                 case -1:
-                    request.status_code_view = '<img class="icon" src="static/icons/math-multiplication-icon.svg-+-$**HTTPDBG_VERSION**$/" alt="load failed"/>';
+                    request.status_code_view = '<svg class="icon" alt="load failed"><use href="#math-multiplication-icon"></use></svg>';
                     break;
                 default:
                     request.status_code_view = request.status_code;
                     break;
             }
         }
-    
-        global.requests[request_id] = request;    
-    
+
+        global.requests[request_id] = request;
+
         if (!request.pin) {
             get_request(request_id);
-        }    
+        }
     }
 }
 
-async function get_all_requests() {
+async function load_all_requests() {
+    if (typeof global.static_all_requests !== "undefined") {
+        return global.static_all_requests;
+    }
 
     var requests_already_loaded = 0
     for (const [request_id, request] of Object.entries(global.requests)) {
@@ -62,15 +65,22 @@ async function get_all_requests() {
         "requests_already_loaded": requests_already_loaded,
     })
 
-    await fetch(url)
-        .then(res => res.json())
+    const res = await fetch(url);
+    const data = await res.json();
+    return data;
+}
+
+
+async function get_all_requests() {
+
+    await load_all_requests()
         .then(data => {
             global.connected = true;
 
             if (data.session.id != global.session) {
-                clean();                
+                clean();
                 global.session = data.session.id;
-                global.sessions[data.session.id] = data.session;                
+                global.sessions[data.session.id] = data.session;
             };
 
             // for the initiators and the groups, we can just save them without any verification
@@ -93,13 +103,21 @@ async function get_all_requests() {
         .catch((error) => {
             global.connected = false;
         });
+}
+
+async function load_request(request_id) {
+    if (typeof global.static_requests !== "undefined") {
+        return global.static_requests[request_id];
+    }
 
+    const res = await fetch("/request/" + request_id);
+    const data = await res.json();
+    return data;
 }
 
 async function get_request(request_id) {
 
-    await fetch("/request/" + request_id)
-        .then(res => res.json())
+    await load_request(request_id)
         .then(data => {
             global.connected = true;
             global.requests[request_id].filter = prepare_for_filter(global.requests[request_id].url);