har2tree 1.31.3__py3-none-any.whl → 1.36.0__py3-none-any.whl

har2tree/har2tree.py

@@ -13,7 +13,9 @@ from functools import wraps, lru_cache
 from io import BytesIO
 from operator import itemgetter
 from pathlib import Path
-from typing import Any, Callable
+from typing import Any, TypedDict
+from collections.abc import Iterator
+from collections.abc import Callable
 from urllib.parse import unquote_plus, urlparse
 
 from .helper import rebuild_url, Har2TreeError, Har2TreeLogAdapter
@@ -58,7 +60,8 @@ def trace_make_subtree_fallback(method: Callable[..., None]) -> Callable[..., No
 
 def trace_make_subtree(method: Callable[..., None]) -> Callable[..., None]:
     @wraps(method)
-    def _impl(self: Any, root: URLNode, nodes_to_attach: list[URLNode] | None=None, dev_debug: bool=False) -> None:
+    def _impl(self: Any, root: URLNode, nodes_to_attach: list[URLNode] | None=None,
+              dev_debug: bool=False, fallback: bool=False) -> None:
         if dev_debug_mode:
             __load_debug_files()
             if dev_debug_url and root.name == dev_debug_url or nodes_to_attach is not None and any(True for u in nodes_to_attach if u.name == dev_debug_url):
@@ -67,7 +70,7 @@ def trace_make_subtree(method: Callable[..., None]) -> Callable[..., None]:
             elif dev_debug_hostname and root.hostname == dev_debug_hostname or nodes_to_attach is not None and any(True for u in nodes_to_attach if u.hostname == dev_debug_hostname):
                 root.logger.warning(f'Debugging Hostname: {dev_debug_hostname}.')
                 dev_debug = True
-        return method(self, root, nodes_to_attach, dev_debug)
+        return method(self, root, nodes_to_attach, dev_debug, fallback)
     return _impl
 
 
@@ -84,6 +87,15 @@ def __load_debug_files() -> None:
             dev_debug_hostname = f.read().strip()
 
 
+# NOTE: Copy from PlaywrightCapture to avoid extra dep
+class FramesResponse(TypedDict, total=False):
+
+    name: str
+    url: str
+    content: str | None
+    children: list[FramesResponse] | None
+
+
 class HarFile():
 
     def __init__(self, harfile: Path, capture_uuid: str):
@@ -115,8 +127,14 @@ class HarFile():
         last_redirect_file = self.path.parent / f'{root_name}.last_redirect.txt'
         if last_redirect_file.is_file():
             with last_redirect_file.open('r') as _lr:
-                self.final_redirect: str = unquote_plus(_lr.read())
-            self._search_final_redirect()
+                last_redirect = unquote_plus(_lr.read())
+            self.final_redirect: str = last_redirect
+            if not self._search_final_redirect():
+                if last_redirect.startswith('chrome') or last_redirect.startswith('about'):
+                    # the capture failed.
+                    pass
+                else:
+                    self.logger.info(f'Final redirect URL from address bar not in tree: {last_redirect}')
         else:
             self.logger.debug('No last_redirect file available.')
             self.final_redirect = ''
@@ -129,6 +147,14 @@ class HarFile():
             self.logger.debug('No cookies file available.')
             self.cookies = []
 
+        framesfile = self.path.parent / f'{root_name}.frames.json'
+        if framesfile.is_file():
+            with framesfile.open() as c:
+                self.frames: FramesResponse = json.load(c)
+        else:
+            self.logger.debug('No frames file available.')
+            self.frames = {}
+
         dlfile = self.path.parent / f'{root_name}.data'
         dlfilename = self.path.parent / f'{root_name}.data.filename'
         self.downloaded_file: BytesIO | None
@@ -169,29 +195,30 @@ class HarFile():
         # Set to false if initial_redirects fails to find the chain.
         self.need_tree_redirects = False
 
-    def _search_final_redirect(self) -> None:
+    def _search_final_redirect(self) -> bool:
         """Try to find the final path to the final redirect without building the tree"""
         for e in self.entries:
             unquoted_url = unquote_plus(e['request']['url'])
             if unquoted_url == self.final_redirect:
-                break
+                return True
             elif unquoted_url.startswith(f'{self.final_redirect}?'):
                 # WARNING: the URL in that file may not be present in the HAR: the query part is stripped by splash
                 self.final_redirect = unquoted_url
-                break
+                return True
         else:
             # Update 2020-04-01: .. but the fragment is not striped so self.final_redirect may not be found
             # Unless we find the entry in the har, we need to search again without the fragment
             if '#' in self.final_redirect:
                 self.final_redirect = self.final_redirect.split('#', 1)[0]
-                self._search_final_redirect()
+                return self._search_final_redirect()
             elif '?' in self.final_redirect:
                 # At this point, we're trying things. The final URL returned by splash may have been changed
                 # in JavaScript and never appear in the HAR. Let's try to find the closest one with the same path
                 self.final_redirect = self.final_redirect.split('?', 1)[0]
-                self._search_final_redirect()
+                return self._search_final_redirect()
             else:
                 self.logger.info(f'Unable to find the final redirect: {self.final_redirect}')
+        return False
 
     @property
     def number_entries(self) -> int:
@@ -306,8 +333,9 @@ class Har2Tree:
         self.pages_root: dict[str, str] = {}
 
         self.all_redirects: list[str] = []
-        self.all_referer: dict[str, list[str]] = defaultdict(list)
-        self.all_initiator_url: dict[str, list[str]] = defaultdict(list)
+        # 2025-11-16: make values of referers and initiators sets because there will be duplicates
+        self.all_referer: dict[str, set[str]] = defaultdict(set)
+        self.all_initiator_url: dict[str, set[str]] = defaultdict(set)
         self._load_url_entries()
 
         # Generate cookies lookup tables
@@ -398,6 +426,65 @@ class Har2Tree:
 
         self.url_tree = self._nodes_list.pop(0)
 
+    def _url_to_local_only_content(self, url: str | None) -> bool:
+        return (url is None
+                or url in ['about:blank', 'about:srcdoc', '']  # not loading anything remotely
+                or url.startswith('data')  # base64 encoded content
+                or url.startswith('chrome-error')  # not in the HAR/tree
+                or url.startswith('blob')  # blobs aren't URLs
+                )
+
+    def _load_iframes(self, current: URLNode, frames: FramesResponse) -> None:
+        if not frames.get('content') or frames['content'] is None:
+            # NOTE: debug stuff, no content makes it pretty useless.
+            if frames.get('url'):
+                if self._url_to_local_only_content(frames['url']):
+                    self.logger.info('Got an empty frame to local content.')
+                else:
+                    u = unquote_plus(frames['url'])
+                    self.logger.warning(f'Got a url ({u}) for the frame, but no content')
+            else:
+                self.logger.info('Got a frame, but no content.')
+            return
+
+        if frames.get('url') and not self._url_to_local_only_content(frames['url']):
+            u = unquote_plus(frames['url'])
+            possible_child_name = {u, u.split('#', 1)[0]}
+            # this url should be in a node directly attached to that one
+            # we need to find that node
+            for child in current.traverse():
+                if child.name in possible_child_name:
+                    self.logger.debug(f'Found URL "{u}".')
+                    # Found the node, adding the content
+                    if not hasattr(child, 'rendered_frame'):
+                        child.rendered_frame = []
+                    child.rendered_frame.append(BytesIO(frames['content'].encode()))
+                    # and mark the node as iframe
+                    child.add_feature('iframe', True)
+                    # if there are children, use that node as parent and call the current method recursvely
+                    if f_children := frames.get('children'):
+                        for f_child in f_children:
+                            self._load_iframes(child, f_child)
+                    break
+            else:
+                # Couldn'd find the node Oo
+                to_print = ', '.join(possible_child_name)
+                children_to_print = ', '.join([child.name for child in current.traverse()])
+                self.logger.warning(f'Unable to find "{to_print}" in the children of "{current.name}" - {children_to_print}')
+        else:
+            self.logger.debug(f'"{current.name}" contains an iFrame.')
+            # No URL, this frame is directly in the parent frame.
+            if not hasattr(current, 'rendered_frame'):
+                current.rendered_frame = []
+            current.rendered_frame.append(BytesIO(frames['content'].encode()))
+            self.logger.debug(f'"{current.name}" has {len(current.rendered_frame)} iFrames.')
+            # and mark the node as iframe
+            current.add_feature('iframe', True)
+            # if there are children, use that node as parent and call the current method recursvely
+            if f_children := frames.get('children'):
+                for f_child in f_children:
+                    self._load_iframes(current, f_child)
+
     @property
     def initial_referer(self) -> str | None:
         '''The referer passed to the first URL in the tree'''
@@ -520,7 +607,7 @@ class Har2Tree:
 
             if hasattr(n, 'initiator_url'):
                 # The HAR file was created by chrome/chromium and we got the _initiator key
-                self.all_initiator_url[n.initiator_url].append(n.name)
+                self.all_initiator_url[n.initiator_url].add(n.name)
 
             if url_entry['startedDateTime'] in self.har.pages_start_times:
                 for page in self.har.pages_start_times[url_entry['startedDateTime']]:
@@ -533,7 +620,7 @@ class Har2Tree:
             if hasattr(n, 'referer') and i > 0:
                 # NOTE 2021-05-14: referer to self are a real thing: url -> POST to self
                 if n.name != n.referer or ('method' in n.request and n.request['method'] == 'POST'):
-                    self.all_referer[n.referer].append(n.name)
+                    self.all_referer[n.referer].add(n.name)
 
             self._nodes_list.append(n)
             self.all_url_requests[n.name].append(n)
@@ -566,12 +653,6 @@ class Har2Tree:
         if node:
             return node[0]
 
-        browser_errors = ['chrome-error', 'about:blank']
-        if self.har.final_redirect and not any(self.har.final_redirect.startswith(r) for r in browser_errors):
-            self.logger.warning(f'Final redirect URL from adress bar not in tree: {self.har.final_redirect}')
-        else:
-            # No final redirect, already logged earlier.
-            pass
         # Just try to get the best guess: first node after JS/HTTP redirects
         curnode = self.url_tree
         while hasattr(curnode, 'redirect') and curnode.redirect:
@@ -620,6 +701,14 @@ class Har2Tree:
             for child_node_hostname, child_nodes_url in sub_roots.items():
                 self.make_hostname_tree(child_nodes_url, child_node_hostname)
 
+    def _all_urlnodes_in_host_tree(self) -> None:
+        # debug: check if all the nodes in the URL tree are in the hostnode tree (they must have an UUID)
+        self.logger.warning('Validating host tree....')
+        for urlnode in self.url_tree.traverse():
+            if not hasattr(urlnode, 'hostnode_uuid'):
+                self.logger.error(f'URL Node not un host tree: {urlnode}')
+        self.logger.warning('host tree validated.')
+
     def make_tree(self) -> URLNode:
         """Build URL and Host trees"""
         self._make_subtree(self.url_tree)
@@ -650,6 +739,15 @@ class Har2Tree:
         # Initialize the hostname tree root
         self.hostname_tree.add_url(self.url_tree)
         self.make_hostname_tree(self.url_tree, self.hostname_tree)
+        if dev_debug_mode:
+            self._all_urlnodes_in_host_tree()
+        if isinstance(self.har.frames, dict):
+            if self.har.frames.get('children') and self.har.frames['children'] is not None:
+                # we have frames in the main one
+                for f_child in self.har.frames['children']:
+                    self._load_iframes(self.rendered_node, f_child)
+        else:
+            self.logger.warning(f'Wrong format for the frames ({type(self.har.frames)}), very old capture.')
         return self.url_tree
 
     @trace_make_subtree_fallback
@@ -667,7 +765,7 @@ class Har2Tree:
                             # we got an non-empty response, breaking
                             break
                     # attach to the the first response with something, or to whatever we get.
-                    self._make_subtree(node_with_hostname, [node])
+                    self._make_subtree(node_with_hostname, [node], fallback=True)
                     return
 
         # Sometimes, the har has a list of pages, generally when we have HTTP redirects.
@@ -679,20 +777,25 @@ class Har2Tree:
                 and node.pageref != self.har.har['log']['pages'][0]
                 and self.pages_root[node.pageref] != node.uuid):
             # In that case, we check if there is already a page with the pageref of the orphan node,
-            # and attach the node to that. NOTE: we can only do that if there is already a node with this pageref in the tree.
+            # and attach the node to that.
+            # NOTE: we can only do that if there is already a node with this pageref in the tree.
             # This node is not a page root, we can attach it \o/
             page_root_node = self.get_url_node_by_uuid(self.pages_root[node.pageref])
             if dev_debug:
                 self.logger.warning(f'Failed to attach URLNode in the normal process, attaching node to page {node.pageref} - Node: {page_root_node.uuid} - {page_root_node.name}.')
-            self._make_subtree(page_root_node, [node])
-        elif self.url_tree.search_nodes(name=self.har.final_redirect):
-            # Generally, when we have a bunch of redirects, they do not branch out before the final landing page
-            # *but* it is not always the case: some intermediary redirects will have calls to 3rd party pages.
+            self._make_subtree(page_root_node, [node], fallback=True)
+        elif self.rendered_node != self.url_tree:
+            # Generally, when we have a bunch of redirects, they (generally) do not branch out
+            # before the final landing page *but* it is not always the case: some intermediary
+            # redirects will have calls to 3rd party pages.
             # Hopefully, this last case was taken care of in the branch above.
-            # In this branch, we get the landing page after the redirects (if any), and attach the node to it.
+            # In this branch, we get the landing page after the redirects, and attach the node to it.
+
+            # We skip this call if there are no redirects as it is the very last fallback at the
+            # end of this method anyway
             if dev_debug:
                 self.logger.warning(f'Failed to attach URLNode in the normal process, attaching node to final redirect: {self.har.final_redirect}.')
-            self._make_subtree(self.url_tree.search_nodes(name=self.har.final_redirect)[0], [node])
+            self._make_subtree(self.rendered_node, [node], fallback=True)
         elif 'pages' in self.har.har['log']:
             # No luck, the node is root for this pageref, let's attach it to the prior page in the list, or the very first node (tree root)
             page_before = self.har.har['log']['pages'][0]
@@ -714,13 +817,38 @@ class Har2Tree:
                 # node to the root node
                 page_root_node = self.url_tree
                 self.logger.warning('The pages in the HAR are in in the wrong order, this should not happen but here we are')
-            self._make_subtree(page_root_node, [node])
+            self._make_subtree(page_root_node, [node], fallback=True)
         else:
             # no way to attach it to anything else, attach to the root node
-            self._make_subtree(self.url_tree, [node])
+            self._make_subtree(self.url_tree, [node], fallback=True)
+
+    def all_real_urls_in_children(self, frame: FramesResponse) -> Iterator[str]:
+        # from a frame, search all the real urls in each of the children, stop at the first one
+        if (frame.get('url') and frame['url'] is not None and not self._url_to_local_only_content(frame['url'])):
+            yield frame['url']
+        else:
+            # got no real URL, try the children
+            if frame.get('children') and frame['children'] is not None:
+                for c in frame['children']:
+                    yield from self.all_real_urls_in_children(c)
+
+    def search_in_frames(self, urls: set[str], frame: FramesResponse) -> Iterator[str]:
+        # If the frame doesn't have children, there are no potential URLs to attach
+        if not isinstance(frame, dict) or not frame.get('children') or frame['children'] is None:
+            return None
+
+        if frame.get('url'):
+            u = unquote_plus(frame['url'])
+            if urls & {u, u.split('#', 1)[0]}:
+                # got a matching URL, get list of potential iframes urls
+                for c in frame['children']:
+                    yield from self.all_real_urls_in_children(c)
+        for c in frame['children']:
+            yield from self.search_in_frames(urls, c)
 
     @trace_make_subtree
-    def _make_subtree(self, root: URLNode, nodes_to_attach: list[URLNode] | None=None, dev_debug: bool=False) -> None:
+    def _make_subtree(self, root: URLNode, nodes_to_attach: list[URLNode] | None=None,
+                      dev_debug: bool=False, fallback: bool=False) -> None:
         """Recursive method building each level of the tree"""
         matching_urls: list[URLNode]
         if nodes_to_attach is None:
@@ -782,6 +910,26 @@ class Har2Tree:
             if unode.empty_response:
                 continue
 
+            # 2025-11-14
+            # the referer of an iframe is the hostname of the parent, even if the parent
+            # is a URL with a full path. Before using the referer, we need to check if we have
+            # the current url in the frame tree. If we do, find nodes (in the remaining list)
+            # with the URLs of the children - any fragment will be missing - and attach that node
+            possible_iframe_urls = {unode.name, unode.name.split('#', 1)[0]}
+            for possible_url in self.search_in_frames(possible_iframe_urls, self.har.frames):
+                cu = unquote_plus(possible_url)
+                for u in {cu, cu.split('#', 1)[0]}:
+                    if u not in self.all_url_requests:
+                        if '#' not in u:
+                            self.logger.info(f'"{u}" in the frames URLs, but not in the HAR.')
+                        continue
+                    matching_urls = [url_node for url_node in self.all_url_requests[u]
+                                     if url_node in self._nodes_list]
+                    self._nodes_list = [node for node in self._nodes_list if node not in matching_urls]
+                    if dev_debug:
+                        self.logger.warning(f'Found via initiator from {unode.name} to {matching_urls}.')
+                    self._make_subtree(unode, matching_urls)
+
             # The node can have a redirect, but also trigger ressources refering to themselves, we need to trigger this code on each node.
             if self.all_initiator_url.get(unode.name):
                 # The URL (unode.name) is in the list of known urls initiating calls
@@ -813,6 +961,12 @@ class Har2Tree:
             if hasattr(unode, 'external_ressources'):
                 # the url loads external things, and some of them have no referer....
                 for external_tag, links in unode.external_ressources.items():
+                    # 2025-11-06: skip full regex until we're calling this method in the fallback
+                    #             the iframes will often (not always) have a referer set and the URL
+                    #             might be found by the regex and it will not be attached at the
+                    #             right place
+                    if external_tag == 'full_regex' and not fallback:
+                        continue
                     for link in links:
                         if link not in self.all_url_requests or link == self.har.final_redirect:
                             # We have a lot of false positives

har2tree/helper.py

@@ -72,7 +72,7 @@ def make_hhhash(entry: dict[str, Any]) -> str:
     # We need the HTTP version used for the query:
     # * The HTTP Header names in HTTP 1.1 can have uppercase characters
     # * The HTTP Header names in HTTP 2 *must* be lowercase: https://www.rfc-editor.org/rfc/rfc7540#section-8.1.2
-    if entry['httpVersion'].lower() in ["http/1.1", "http/1.0"]:
+    if entry['httpVersion'].lower() in ["http/1.1", "http/1.0", "1.1"]:
         return f'hhh:1:{sha256}'
     if entry['httpVersion'].lower() == "http/2.0":
         return f'hhh:2:{sha256}'
@@ -364,7 +364,8 @@ def find_external_ressources(mimetype: str, data: bytes, base_url: str, all_requ
     # link: https://www.w3schools.com/TAGs/tag_link.asp -> href
     # object: https://www.w3schools.com/TAGs/tag_object.asp -> data
     external_ressources: dict[str, list[str]] = {'img': [], 'script': [], 'video': [], 'audio': [],
-                                                 'iframe': [], 'embed': [], 'source': [],
+                                                 'iframe': [],
+                                                 'embed': [], 'source': [],
                                                  'link': [],
                                                  'object': [],
                                                  'css': [],

har2tree/nodes.py

@@ -15,16 +15,19 @@ from base64 import b64decode
 from datetime import datetime, timedelta
 from functools import lru_cache, cached_property
 from hashlib import sha256
-from io import BytesIO
+from io import BytesIO, StringIO
 from pathlib import Path
 from typing import Any
 from collections.abc import MutableMapping
-from urllib.parse import unquote_plus, urlparse, urljoin
+from urllib.parse import unquote_plus, urlparse, urljoin, parse_qs
 
 import filetype  # type: ignore
+import json_stream  # type: ignore
+
 from bs4 import BeautifulSoup
 from ete3 import TreeNode  # type: ignore
 from publicsuffixlist import PublicSuffixList  # type: ignore
+from requests_toolbelt.multipart import decoder  # type: ignore
 from w3lib.html import strip_html5_whitespace
 from w3lib.url import canonicalize_url, safe_url_string
 
@@ -211,33 +214,67 @@ class URLNode(HarTreeNode):
         if 'user_agent' not in self.features:
             self.add_feature('user_agent', '')
 
-        if 'method' in self.request and self.request['method'] == 'POST' and 'postData' in self.request:
-            # If the content is empty, we don't care
-            if self.request['postData']['text']:
-                _posted_data: str = self.request['postData']['text']
-                decoded_posted_data: str | bytes | int | float | bool
-                # NOTE 2023-08-22: Blind attempt to base64 decode the data
+        if 'method' in self.request and self.request['method'] == 'POST':
+            decoded_posted_data: list[Any] | str | bytes | int | float | bool | dict[str, str] | dict[str, list[str]] | None = None
+            if 'postData' not in self.request or 'text' not in self.request['postData']:
+                self.logger.debug('POST request with no content.')
+                self.add_feature('posted_data_info', "No content.")
+            elif not self.request['postData']['text']:
+                # If the POST content is empty
+                self.logger.debug('Empty POST request.')
+                decoded_posted_data = ''
+                self.add_feature('posted_data_info', "Empty request.")
+            elif self.request['postData']['text'].startswith('\x1f\uFFFD\x08'):
+                # b'\x1f\xef\xbf\xbd\x08', decoded to UTF-8
+                # => the replacement character
+                # https://www.cogsci.ed.ac.uk/~richard/utf-8.cgi?input=%EF%BF%BD&mode=char
+                self.logger.debug('Got a garbled gzipped POST blob.')
+                self.add_feature('posted_data_info', "It was a POSTed gzipped blob, but the data has been garbled.")
+                decoded_posted_data = self.request['postData']['text']
+            elif self.request['postData'].get('params'):
+                # NOTE 2025-08-08
+                # if the posted data mimetype is "application/x-www-form-urlencoded"
+                # the HAR contains the decoded entry in the params key
+                # The params key is a list of dicts with a key and a value
+                #   {"name": <key>, "value": <data>}
+                # I'd rather have it as {<key>: <data>}
+                # TODO: some processing on the data part (it's often a json blob)
+                self.logger.debug('Got a params POST.')
+                decoded_posted_data = {entry['name']: entry['value'] for entry in self.request['postData']['params']}
+                self.add_feature('posted_data_info', "POST request as URL params.")
+            else:
+                self.logger.debug('Got a normal POST')
                 try:
-                    decoded_posted_data = self._dirty_safe_b64decode(_posted_data)
+                    # NOTE 2023-08-22: Blind attempt to base64 decode the data
+                    decoded_posted_data = self._dirty_safe_b64decode(self.request['postData']['text'])
                 except binascii.Error:
-                    decoded_posted_data = _posted_data
+                    decoded_posted_data = self.request['postData']['text']
                 if 'mimeType' in self.request['postData']:
                     # make it easier to compare.
                     mimetype_lower = self.request['postData']['mimeType'].lower()
                     if mimetype_lower.startswith('application/x-www-form-urlencoded'):
+                        # NOTE: this should never happen as there should
+                        # be something in self.request['postData']['params']
+                        # and we already processed it before but just in case...
+                        self.logger.debug('Got a application/x-www-form-urlencoded without params key')
                         # 100% sure there will be websites where decode will fail
                         try:
                             if isinstance(decoded_posted_data, bytes):
                                 decoded_posted_data = decoded_posted_data.decode()
                             if isinstance(decoded_posted_data, str):
                                 decoded_posted_data = unquote_plus(decoded_posted_data)
+                            if isinstance(decoded_posted_data, str):
+                                decoded_posted_data = parse_qs(decoded_posted_data)
+                            self.add_feature('posted_data_info', "Successfully decoded POST request.")
                         except Exception as e:
-                            self.logger.warning(f'Unable to unquote form data "{decoded_posted_data!r}": {e}')
+                            self.logger.warning(f'Unable to unquote or parse form data "{decoded_posted_data!r}": {e}')
+                            self.add_feature('posted_data_info', "Unable to decode POST request.")
                     elif (mimetype_lower.startswith('application/json')
                           or mimetype_lower.startswith('application/csp-report')
                           or mimetype_lower.startswith('application/x-amz-json-1.1')
-                          or mimetype_lower.startswith('application/x-json-stream')
                           or mimetype_lower.startswith('application/reports+json')
+                          or mimetype_lower.startswith('application/vnd.adobe.dc+json')
+                          or mimetype_lower.startswith('application/ion+json')
                           or mimetype_lower.endswith('json')
                           ):
                         if isinstance(decoded_posted_data, (str, bytes)):
@@ -245,56 +282,127 @@ class URLNode(HarTreeNode):
                             try:
                                 # NOTE 2023-08-22: loads here may give us a int, float or a bool.
                                 decoded_posted_data = json.loads(decoded_posted_data)
+                                self.add_feature('posted_data_info', "Successfully decoded POST request.")
                             except Exception:
+                                self.add_feature('posted_data_info', "Unable to decode POST request.")
                                 if isinstance(decoded_posted_data, (str, bytes)):
-                                    self.logger.debug(f"Expected json, got garbage: {mimetype_lower} - {decoded_posted_data[:20]!r}[...]")
+                                    self.logger.warning(f"Expected json, got garbage: {mimetype_lower} - {decoded_posted_data[:20]!r}[...]")
                                 else:
-                                    self.logger.debug(f"Expected json, got garbage: {mimetype_lower} - {decoded_posted_data}")
+                                    self.logger.warning(f"Expected json, got garbage: {mimetype_lower} - {decoded_posted_data}")
+                    elif mimetype_lower.startswith('application/x-json-stream'):
+                        try:
+                            to_stream: StringIO | BytesIO
+                            if isinstance(decoded_posted_data, str):
+                                to_stream = StringIO(decoded_posted_data)
+                            elif isinstance(decoded_posted_data, bytes):
+                                to_stream = BytesIO(decoded_posted_data)
+                            else:
+                                raise ValueError(f'Invalid type: {type(decoded_posted_data)}')
+                            streamed_data = json_stream.load(to_stream)
+                            decoded_posted_data = json_stream.to_standard_types(streamed_data)
+                            self.add_feature('posted_data_info', "Successfully decoded POST request.")
+                        except Exception:
+                            if isinstance(decoded_posted_data, (str, bytes)):
+                                self.logger.warning(f"Expected json stream, got garbage: {mimetype_lower} - {decoded_posted_data[:20]!r}[...]")
+                            else:
+                                self.logger.warning(f"Expected json stream, got garbage: {mimetype_lower} - {decoded_posted_data}")
+                            self.add_feature('posted_data_info', "Unable to decode POST request.")
+                    elif mimetype_lower.startswith('multipart'):
+                        self.add_feature('posted_data_info', f"Decoding {mimetype_lower} is partially supported.")
+                        if isinstance(decoded_posted_data, str):
+                            # must be encoded for decoding
+                            multipart_to_decode = decoded_posted_data.encode()
+                        elif isinstance(decoded_posted_data, bytes):
+                            multipart_to_decode = decoded_posted_data
+                        else:
+                            raise ValueError(f'Invalid type for multipart POST: {type(decoded_posted_data)}')
+                        if b"\r\n" not in multipart_to_decode:
+                            # the decoder wants that
+                            multipart_to_decode = multipart_to_decode.replace(b"\n", b"\r\n")
+                        try:
+                            multipart_data = decoder.MultipartDecoder(multipart_to_decode, mimetype_lower)
+                            decoded_posted_data = []
+                            for part in multipart_data.parts:
+                                headers = {k.decode(): v.decode() for k, v in part.headers.items()}
+                                content = part.text
+                                decoded_posted_data.append({'headers': headers, 'content': content})
+                        except Exception as e:
+                            self.logger.warning(f'Unable to decode multipart POST: {e}')
+                            self.add_feature('posted_data_info', "Unable to decode multipart in POST request.")
 
-                    elif mimetype_lower.startswith('multipart/form-data'):
-                        # FIXME multipart content (similar to email). Not totally sure what do do with it tight now.
-                        pass
                     elif mimetype_lower.startswith('application/x-protobuf'):
                         # FIXME If possible, decode?
-                        pass
-                    elif mimetype_lower.startswith('text'):
+                        self.logger.debug(f'Got a POST {mimetype_lower}: {decoded_posted_data!r}')
+                        self.add_feature('posted_data_info', f"Decoding {mimetype_lower} is not supported yet.")
+                    elif mimetype_lower.startswith('text') and isinstance(decoded_posted_data, (str, bytes)):
                         try:
                             # NOTE 2023-08-22: Quite a few text entries are in fact json, give it a shot.
                             # loads here may give us a int, float or a bool.
                             decoded_posted_data = json.loads(decoded_posted_data)
+                            self.add_feature('posted_data_info', "Decoded JSON out of POST request.")
                         except Exception:
                             # keep it as it is otherwise.
                             pass
                     elif mimetype_lower.endswith('javascript'):
                         # keep it as it is
-                        pass
-                    elif mimetype_lower == '?':
-                        # Just skip it, no need to go in the warnings
-                        pass
-                    elif mimetype_lower in ['application/octet-stream', 'application/binary']:
+                        self.logger.warning(f'Got a POST {mimetype_lower}: {decoded_posted_data!r}')
+                        self.add_feature('posted_data_info', f"Pretty rendering of {mimetype_lower} is not supported yet.")
+                    elif mimetype_lower in ['?', '*/*']:
+                        self.logger.warning(f'Got a POST {mimetype_lower}: {decoded_posted_data!r}')
+                        self.add_feature('posted_data_info', f"Weird MimeType ({mimetype_lower}) is not supported yet.")
+                    elif mimetype_lower == 'application/binary':
+                        self.logger.warning(f'Got a POST {mimetype_lower}, not a broken gziped blob: {decoded_posted_data!r}')
+                        self.add_feature('posted_data_info', f"MimeType ({mimetype_lower}) is not supported yet.")
+                    elif mimetype_lower in ['application/octet-stream']:
                         # Should flag it, maybe?
-                        pass
-                    elif mimetype_lower in ['application/unknown', 'application/grpc-web+proto']:
+                        self.logger.warning(f'Got a POST {mimetype_lower}: {decoded_posted_data!r}')
+                        self.add_feature('posted_data_info', f"MimeType ({mimetype_lower}) is not supported yet.")
+                    elif mimetype_lower in ['application/grpc-web+proto']:
+                        # Can be decoded?
+                        self.logger.warning(f'Got a POST {mimetype_lower} - can be decoded: {decoded_posted_data!r}')
+                        self.add_feature('posted_data_info', f"MimeType ({mimetype_lower}) is not supported yet.")
+                    elif mimetype_lower in ['application/unknown']:
                         # Weird but already seen stuff
-                        pass
+                        self.logger.warning(f'Got a POST {mimetype_lower}: {decoded_posted_data!r}')
+                        self.add_feature('posted_data_info', f"MimeType ({mimetype_lower}) is not supported yet.")
                     else:
-                        self.logger.warning(f'Unexpected mime type: {mimetype_lower}')
-
-                # NOTE 2023-08-22: Blind attempt to process the data as json
-                if isinstance(decoded_posted_data, (str, bytes)):
-                    try:
-                        decoded_posted_data = json.loads(decoded_posted_data)
-                    except Exception:
-                        pass
-
-                if isinstance(decoded_posted_data, bytes):
-                    # NOTE 2023-08-22: Blind attempt to decode the bytes
-                    # Try to decode it as utf-8
-                    try:
-                        decoded_posted_data = decoded_posted_data.decode('utf-8')
-                    except Exception:
-                        pass
-                self.add_feature('posted_data', decoded_posted_data)
+                        self.logger.warning(f'Unexpected mime type: {mimetype_lower} - {decoded_posted_data!r}')
+                        self.add_feature('posted_data_info', f"Unexpected MimeType ({mimetype_lower}) is not supported yet.")
+                else:
+                    self.logger.warning(f'Missing mimetype in POST: {self.request["postData"]}')
+                    self.add_feature('posted_data_info', "Missing MimeType, not sure what to do.")
+
+            # NOTE 2023-08-22: Blind attempt to process the data as json
+            if decoded_posted_data and isinstance(decoded_posted_data, (str, bytes)):
+                try:
+                    decoded_posted_data = json.loads(decoded_posted_data)
+                except Exception:
+                    pass
+
+            if decoded_posted_data and isinstance(decoded_posted_data, bytes):
+                # NOTE 2023-08-22: Blind attempt to decode the bytes
+                # Try to decode it as utf-8
+                try:
+                    decoded_posted_data = decoded_posted_data.decode('utf-8')
+                except Exception:
+                    pass
+
+            self.add_feature('posted_data', decoded_posted_data)
+            if 'postData' in self.request and self.request['postData'].get('mimeType'):
+                self.add_feature('posted_data_mimetype', self.request['postData']['mimeType'])
+            # Get size, post decode.
+            if not decoded_posted_data:
+                # empty or None, set to 0
+                self.add_feature('posted_data_size', 0)
+            elif isinstance(decoded_posted_data, (list, dict)):
+                # set size to the json dump
+                self.add_feature('posted_data_size', len(json.dumps(decoded_posted_data)))
+            elif isinstance(decoded_posted_data, (str, bytes)):
+                # length
+                self.add_feature('posted_data_size', len(decoded_posted_data))
+            else:
+                # Stringify and len
+                self.add_feature('posted_data_size', len(str(decoded_posted_data)))
 
         self.add_feature('response', har_entry['response'])
         try:
@@ -377,6 +485,7 @@ class URLNode(HarTreeNode):
 
             # Common JS redirect we can catch easily
             # NOTE: it is extremely fragile and doesn't work very often but is kinda better than nothing.
+            # NOTE 2025-08-30: Also, finding that doesn't mean it is in a part of the code that is executed without user interaction. It can be triggered after a user fills a form for example.
             # Source: https://stackoverflow.com/questions/13363174/regular-expression-to-catch-as-many-javascript-redirections-as-possible
             regex = re.compile(br"""((location.href)|(window.location)|(location.replace)|(location.assign))(( ?= ?)|( ?\( ?))("|')([^'"]*)("|')( ?\) ?)?;""", re.I)
             matches = re.findall(regex, self.body.getvalue())
@@ -478,7 +587,7 @@ class URLNode(HarTreeNode):
             return href
 
         if not hasattr(self, 'rendered_html') or not self.rendered_html:
-            raise Har2TreeError('Not the node of a page rendered, invalid request.')
+            raise Har2TreeError(f'Not the node of a page rendered ({self.uuid}), invalid request.')
         urls: set[str] = set()
 
         # The simple ones: the links.

{har2tree-1.31.3.dist-info → har2tree-1.36.0.dist-info}/METADATA

@@ -1,36 +1,38 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: har2tree
-Version: 1.31.3
+Version: 1.36.0
 Summary: HTTP Archive (HAR) to ETE Toolkit generator
-License: BSD-3-Clause
+License-Expression: BSD-3-Clause
+License-File: LICENSE
 Author: Raphaël Vinot
 Author-email: raphael.vinot@circl.lu
-Requires-Python: >=3.9
+Requires-Python: >=3.10,<3.15
 Classifier: Intended Audience :: Information Technology
 Classifier: Intended Audience :: Science/Research
 Classifier: Intended Audience :: Telecommunications Industry
-Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Internet
 Classifier: Topic :: Security
 Provides-Extra: docs
-Requires-Dist: Sphinx (>=8.2.3) ; (python_version >= "3.11") and (extra == "docs")
-Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.13.4)
+Requires-Dist: Sphinx (>=9.0.4) ; (python_version >= "3.11") and (extra == "docs")
+Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.14.3)
 Requires-Dist: ete3 (>=3.1.3)
 Requires-Dist: filetype (>=1.2.0)
-Requires-Dist: legacy-cgi (>=2.6.3) ; python_version >= "3.13,<4.0"
-Requires-Dist: numpy (<2.1) ; python_version < "3.10"
-Requires-Dist: numpy (<2.3) ; python_version < "3.11"
-Requires-Dist: numpy (>=2.3.1) ; python_version >= "3.11"
-Requires-Dist: publicsuffixlist (>=1.0.2.20250719)
+Requires-Dist: json-stream (>=2.3.3,<3.0.0)
+Requires-Dist: legacy-cgi (>=2.6.4) ; python_version >= "3.13" and python_version < "4.0"
+Requires-Dist: multipart (>=1.3.0,<2.0.0)
+Requires-Dist: numpy (>=2.2,<2.3) ; python_version < "3.11"
+Requires-Dist: numpy (>=2.3.5) ; python_version >= "3.11" and python_version < "3.15"
+Requires-Dist: publicsuffixlist (>=1.0.2.20251209)
+Requires-Dist: requests-toolbelt (>=1.0.0,<2.0.0)
 Requires-Dist: six (>=1.17.0) ; extra == "docs"
-Requires-Dist: tinycss2 (>=1.4.0)
+Requires-Dist: tinycss2 (>=1.5.1)
 Requires-Dist: w3lib (>=2.3.1)
 Project-URL: Documentation, https://har2tree.readthedocs.io/en/latest/
 Project-URL: Repository, https://github.com/Lookyloo/har2tree

har2tree-1.36.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+har2tree/__init__.py,sha256=Na3mxHkUBq3rzYbxiLNJF37DxH5mcghSorjzXw5Teug,422
+har2tree/har2tree.py,sha256=24Puk4dlDXWOVIAPV7SIXNoP-oP-_7ERH2mZPxXiwn8,52762
+har2tree/helper.py,sha256=ktX5Fq-K_t4r0VVAXIH4uy7xc-qCjtSaiUvkX_PYxhw,20737
+har2tree/nodes.py,sha256=QWKqEUnuW7J6pASVvzwWAQNqL-_KDzSs2ld6uJl3qbw,37710
+har2tree/parser.py,sha256=4yej1OcVYAIiLfzYZsO9WCw3WyM_ykDTuvpW7UO1ROE,3645
+har2tree/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+har2tree-1.36.0.dist-info/METADATA,sha256=jjZ2lxWFYv19ZpFL_1ehSNbUNLqEePYGNmLOABeUdnM,2240
+har2tree-1.36.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+har2tree-1.36.0.dist-info/licenses/LICENSE,sha256=Xa4EVROgJsEo10CW-ISCRiw0TtqdKz1JuM3BBLBM55c,1803
+har2tree-1.36.0.dist-info/RECORD,,

{har2tree-1.31.3.dist-info → har2tree-1.36.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.3
+Generator: poetry-core 2.2.1
 Root-Is-Purelib: true
 Tag: py3-none-any

har2tree-1.31.3.dist-info/RECORD

@@ -1,10 +0,0 @@
-har2tree/__init__.py,sha256=Na3mxHkUBq3rzYbxiLNJF37DxH5mcghSorjzXw5Teug,422
-har2tree/har2tree.py,sha256=47x9X5tY69f9SXkYJgJsnAaX2kxgXHgzFThGz6M86Zw,44495
-har2tree/helper.py,sha256=CgeXqfBeHs8SbkW7TRNKqJBTZLAu63KggQjbGHCZAGI,20681
-har2tree/nodes.py,sha256=CC3NseEaM455JOpPqjfTAQ-dwWiGWmzlceGSSeTwoRo,28951
-har2tree/parser.py,sha256=4yej1OcVYAIiLfzYZsO9WCw3WyM_ykDTuvpW7UO1ROE,3645
-har2tree/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-har2tree-1.31.3.dist-info/LICENSE,sha256=Xa4EVROgJsEo10CW-ISCRiw0TtqdKz1JuM3BBLBM55c,1803
-har2tree-1.31.3.dist-info/METADATA,sha256=PSDu0bnPUYje8It-uyZfcDVbo4TTTC7RCzH-2CRAc0U,2112
-har2tree-1.31.3.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-har2tree-1.31.3.dist-info/RECORD,,