har2tree 1.34.1__tar.gz → 1.35.0__tar.gz

{har2tree-1.34.1 → har2tree-1.35.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: har2tree
-Version: 1.34.1
+Version: 1.35.0
 Summary: HTTP Archive (HAR) to ETE Toolkit generator
 License-Expression: BSD-3-Clause
 License-File: LICENSE
@@ -28,8 +28,8 @@ Requires-Dist: json-stream (>=2.3.3,<3.0.0)
 Requires-Dist: legacy-cgi (>=2.6.4) ; python_version >= "3.13" and python_version < "4.0"
 Requires-Dist: multipart (>=1.3.0,<2.0.0)
 Requires-Dist: numpy (>=2.2,<2.3) ; python_version < "3.11"
-Requires-Dist: numpy (>=2.3.4) ; python_version >= "3.11" and python_version < "4.0"
-Requires-Dist: publicsuffixlist (>=1.0.2.20251107)
+Requires-Dist: numpy (>=2.3.5) ; python_version >= "3.11" and python_version < "4.0"
+Requires-Dist: publicsuffixlist (>=1.0.2.20251115)
 Requires-Dist: requests-toolbelt (>=1.0.0,<2.0.0)
 Requires-Dist: six (>=1.17.0) ; extra == "docs"
 Requires-Dist: tinycss2 (>=1.4.0)

{har2tree-1.34.1 → har2tree-1.35.0}/har2tree/har2tree.py

@@ -14,6 +14,7 @@ from io import BytesIO
 from operator import itemgetter
 from pathlib import Path
 from typing import Any, TypedDict
+from collections.abc import Iterator
 from collections.abc import Callable
 from urllib.parse import unquote_plus, urlparse
 
@@ -332,8 +333,9 @@ class Har2Tree:
         self.pages_root: dict[str, str] = {}
 
         self.all_redirects: list[str] = []
-        self.all_referer: dict[str, list[str]] = defaultdict(list)
-        self.all_initiator_url: dict[str, list[str]] = defaultdict(list)
+        # 2025-11-16: make values of referers and initiators sets because there will be duplicates
+        self.all_referer: dict[str, set[str]] = defaultdict(set)
+        self.all_initiator_url: dict[str, set[str]] = defaultdict(set)
         self._load_url_entries()
 
         # Generate cookies lookup tables
@@ -424,12 +426,20 @@ class Har2Tree:
 
         self.url_tree = self._nodes_list.pop(0)
 
+    def _url_to_local_only_content(self, url: str | None) -> bool:
+        return (url is None
+                or url in ['about:blank', 'about:srcdoc', '']  # not loading anything remotely
+                or url.startswith('data')  # base64 encoded content
+                or url.startswith('chrome-error')  # not in the HAR/tree
+                or url.startswith('blob')  # blobs aren't URLs
+                )
+
     def _load_iframes(self, current: URLNode, frames: FramesResponse) -> None:
         if not frames.get('content') or frames['content'] is None:
             # NOTE: debug stuff, no content makes it pretty useless.
             if frames.get('url'):
-                if frames['url'] == "about:blank":
-                    self.logger.info('Got a frame to about:blank with no content.')
+                if self._url_to_local_only_content(frames['url']):
+                    self.logger.info('Got an empty frame to local content.')
                 else:
                     u = unquote_plus(frames['url'])
                     self.logger.warning(f'Got a url ({u}) for the frame, but no content')
@@ -437,12 +447,13 @@ class Har2Tree:
                 self.logger.info('Got a frame, but no content.')
             return
 
-        if frames.get('url') and not (frames['url'] in ['about:blank']):
+        if frames.get('url') and not self._url_to_local_only_content(frames['url']):
             u = unquote_plus(frames['url'])
+            possible_child_name = {u, u.split('#', 1)[0]}
             # this url should be in a node directly attached to that one
             # we need to find that node
             for child in current.traverse():
-                if child.name in [u, u.split('#', 1)[0]]:
+                if child.name in possible_child_name:
                     self.logger.debug(f'Found URL "{u}".')
                     # Found the node, adding the content
                     if not hasattr(child, 'rendered_frame'):
@@ -457,7 +468,9 @@ class Har2Tree:
                     break
             else:
                 # Couldn'd find the node Oo
-                self.logger.warning(f'Unable to find "{u}" in the children of "{current.name}"')
+                to_print = ', '.join(possible_child_name)
+                children_to_print = ', '.join([child.name for child in current.traverse()])
+                self.logger.warning(f'Unable to find "{to_print}" in the children of "{current.name}" - {children_to_print}')
         else:
             self.logger.debug(f'"{current.name}" contains an iFrame.')
             # No URL, this frame is directly in the parent frame.
@@ -594,7 +607,7 @@ class Har2Tree:
 
             if hasattr(n, 'initiator_url'):
                 # The HAR file was created by chrome/chromium and we got the _initiator key
-                self.all_initiator_url[n.initiator_url].append(n.name)
+                self.all_initiator_url[n.initiator_url].add(n.name)
 
             if url_entry['startedDateTime'] in self.har.pages_start_times:
                 for page in self.har.pages_start_times[url_entry['startedDateTime']]:
@@ -607,7 +620,7 @@ class Har2Tree:
             if hasattr(n, 'referer') and i > 0:
                 # NOTE 2021-05-14: referer to self are a real thing: url -> POST to self
                 if n.name != n.referer or ('method' in n.request and n.request['method'] == 'POST'):
-                    self.all_referer[n.referer].append(n.name)
+                    self.all_referer[n.referer].add(n.name)
 
             self._nodes_list.append(n)
             self.all_url_requests[n.name].append(n)
@@ -728,10 +741,13 @@ class Har2Tree:
         self.make_hostname_tree(self.url_tree, self.hostname_tree)
         if dev_debug_mode:
             self._all_urlnodes_in_host_tree()
-        if self.har.frames.get('children') and self.har.frames['children'] is not None:
-            # we have frames in the main one
-            for f_child in self.har.frames['children']:
-                self._load_iframes(self.rendered_node, f_child)
+        if isinstance(self.har.frames, dict):
+            if self.har.frames.get('children') and self.har.frames['children'] is not None:
+                # we have frames in the main one
+                for f_child in self.har.frames['children']:
+                    self._load_iframes(self.rendered_node, f_child)
+        else:
+            self.logger.warning(f'Wrong format for the frames ({type(self.har.frames)}), very old capture.')
         return self.url_tree
 
     @trace_make_subtree_fallback
@@ -806,6 +822,30 @@ class Har2Tree:
             # no way to attach it to anything else, attach to the root node
             self._make_subtree(self.url_tree, [node], fallback=True)
 
+    def all_real_urls_in_children(self, frame: FramesResponse) -> Iterator[str]:
+        # from a frame, search all the real urls in each of the children, stop at the first one
+        if (frame.get('url') and frame['url'] is not None and not self._url_to_local_only_content(frame['url'])):
+            yield frame['url']
+        else:
+            # got no real URL, try the children
+            if frame.get('children') and frame['children'] is not None:
+                for c in frame['children']:
+                    yield from self.all_real_urls_in_children(c)
+
+    def search_in_frames(self, urls: set[str], frame: FramesResponse) -> Iterator[str]:
+        # If the frame doesn't have children, there are no potential URLs to attach
+        if not frame.get('children') or frame['children'] is None:
+            return None
+
+        if frame.get('url'):
+            u = unquote_plus(frame['url'])
+            if urls & {u, u.split('#', 1)[0]}:
+                # got a matching URL, get list of potential iframes urls
+                for c in frame['children']:
+                    yield from self.all_real_urls_in_children(c)
+        for c in frame['children']:
+            yield from self.search_in_frames(urls, c)
+
     @trace_make_subtree
     def _make_subtree(self, root: URLNode, nodes_to_attach: list[URLNode] | None=None,
                       dev_debug: bool=False, fallback: bool=False) -> None:
@@ -870,6 +910,26 @@ class Har2Tree:
             if unode.empty_response:
                 continue
 
+            # 2025-11-14
+            # the referer of an iframe is the hostname of the parent, even if the parent
+            # is a URL with a full path. Before using the referer, we need to check if we have
+            # the current url in the frame tree. If we do, find nodes (in the remaining list)
+            # with the URLs of the children - any fragment will be missing - and attach that node
+            possible_iframe_urls = {unode.name, unode.name.split('#', 1)[0]}
+            for possible_url in self.search_in_frames(possible_iframe_urls, self.har.frames):
+                cu = unquote_plus(possible_url)
+                for u in {cu, cu.split('#', 1)[0]}:
+                    if u not in self.all_url_requests:
+                        if '#' not in u:
+                            self.logger.info(f'"{u}" in the frames URLs, but not in the HAR.')
+                        continue
+                    matching_urls = [url_node for url_node in self.all_url_requests[u]
+                                     if url_node in self._nodes_list]
+                    self._nodes_list = [node for node in self._nodes_list if node not in matching_urls]
+                    if dev_debug:
+                        self.logger.warning(f'Found via initiator from {unode.name} to {matching_urls}.')
+                    self._make_subtree(unode, matching_urls)
+
             # The node can have a redirect, but also trigger ressources refering to themselves, we need to trigger this code on each node.
             if self.all_initiator_url.get(unode.name):
                 # The URL (unode.name) is in the list of known urls initiating calls

{har2tree-1.34.1 → har2tree-1.35.0}/har2tree/helper.py

@@ -72,7 +72,7 @@ def make_hhhash(entry: dict[str, Any]) -> str:
     # We need the HTTP version used for the query:
     # * The HTTP Header names in HTTP 1.1 can have uppercase characters
     # * The HTTP Header names in HTTP 2 *must* be lowercase: https://www.rfc-editor.org/rfc/rfc7540#section-8.1.2
-    if entry['httpVersion'].lower() in ["http/1.1", "http/1.0"]:
+    if entry['httpVersion'].lower() in ["http/1.1", "http/1.0", "1.1"]:
         return f'hhh:1:{sha256}'
     if entry['httpVersion'].lower() == "http/2.0":
         return f'hhh:2:{sha256}'

{har2tree-1.34.1 → har2tree-1.35.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "har2tree"
-version = "1.34.1"
+version = "1.35.0"
 description = "HTTP Archive (HAR) to ETE Toolkit generator"
 authors = [
     {name="Raphaël Vinot", email="raphael.vinot@circl.lu"}
@@ -14,12 +14,12 @@ dynamic = [ "classifiers" ]
 dependencies = [
     "ete3 (>=3.1.3)",
     "beautifulsoup4[charset-normalizer,lxml] (>=4.14.2)",
-    "publicsuffixlist (>=1.0.2.20251107)",
+    "publicsuffixlist (>=1.0.2.20251115)",
     "filetype (>=1.2.0)",
     # poetry up fails with the version of numpy forced for python < 3.11.
     # The work around is to comment it, run poetry up, uncomment it. and run poetry update.
     "numpy (>=2.2,<2.3) ; python_version < '3.11'",
-    "numpy (>=2.3.4) ; python_version >= \"3.11\" and python_version < \"4.0\"",
+    "numpy (>=2.3.5) ; python_version >= \"3.11\" and python_version < \"4.0\"",
     "w3lib (>=2.3.1)",
     "tinycss2 (>=1.4.0)",
     "legacy-cgi (>=2.6.4) ; python_version >= \"3.13\" and python_version < \"4.0\"",
@@ -49,7 +49,7 @@ docs = ["Sphinx (>=8.2.3) ; python_version >= \"3.11\"", "six (>=1.17.0)"]
 [tool.poetry.group.dev.dependencies]
 mypy = "^1.18.2"
 pytest-cov = "^7.0.0"
-coverage = "^7.11.1"
+coverage = "^7.11.3"
 types-beautifulsoup4 = "^4.12.0.20250516"
 
 [build-system]