har2tree 1.31.5__py3-none-any.whl → 1.32.0__py3-none-any.whl

har2tree/har2tree.py

@@ -115,8 +115,10 @@ class HarFile():
         last_redirect_file = self.path.parent / f'{root_name}.last_redirect.txt'
         if last_redirect_file.is_file():
             with last_redirect_file.open('r') as _lr:
-                self.final_redirect: str = unquote_plus(_lr.read())
-            self._search_final_redirect()
+                last_redirect = unquote_plus(_lr.read())
+            self.final_redirect: str = last_redirect
+            if not self._search_final_redirect():
+                self.logger.warning(f'Final redirect URL from address bar not in tree: {last_redirect}')
         else:
             self.logger.debug('No last_redirect file available.')
             self.final_redirect = ''
@@ -169,29 +171,30 @@ class HarFile():
         # Set to false if initial_redirects fails to find the chain.
         self.need_tree_redirects = False
 
-    def _search_final_redirect(self) -> None:
+    def _search_final_redirect(self) -> bool:
         """Try to find the final path to the final redirect without building the tree"""
         for e in self.entries:
             unquoted_url = unquote_plus(e['request']['url'])
             if unquoted_url == self.final_redirect:
-                break
+                return True
             elif unquoted_url.startswith(f'{self.final_redirect}?'):
                 # WARNING: the URL in that file may not be present in the HAR: the query part is stripped by splash
                 self.final_redirect = unquoted_url
-                break
+                return True
         else:
             # Update 2020-04-01: .. but the fragment is not striped so self.final_redirect may not be found
             # Unless we find the entry in the har, we need to search again without the fragment
             if '#' in self.final_redirect:
                 self.final_redirect = self.final_redirect.split('#', 1)[0]
-                self._search_final_redirect()
+                return self._search_final_redirect()
             elif '?' in self.final_redirect:
                 # At this point, we're trying things. The final URL returned by splash may have been changed
                 # in JavaScript and never appear in the HAR. Let's try to find the closest one with the same path
                 self.final_redirect = self.final_redirect.split('?', 1)[0]
-                self._search_final_redirect()
+                return self._search_final_redirect()
             else:
                 self.logger.info(f'Unable to find the final redirect: {self.final_redirect}')
+        return False
 
     @property
     def number_entries(self) -> int:
@@ -566,12 +569,6 @@ class Har2Tree:
         if node:
             return node[0]
 
-        browser_errors = ['chrome-error', 'about:blank']
-        if self.har.final_redirect and not any(self.har.final_redirect.startswith(r) for r in browser_errors):
-            self.logger.warning(f'Final redirect URL from adress bar not in tree: {self.har.final_redirect}')
-        else:
-            # No final redirect, already logged earlier.
-            pass
         # Just try to get the best guess: first node after JS/HTTP redirects
         curnode = self.url_tree
         while hasattr(curnode, 'redirect') and curnode.redirect:
@@ -679,20 +676,25 @@ class Har2Tree:
                 and node.pageref != self.har.har['log']['pages'][0]
                 and self.pages_root[node.pageref] != node.uuid):
             # In that case, we check if there is already a page with the pageref of the orphan node,
-            # and attach the node to that. NOTE: we can only do that if there is already a node with this pageref in the tree.
+            # and attach the node to that.
+            # NOTE: we can only do that if there is already a node with this pageref in the tree.
             # This node is not a page root, we can attach it \o/
             page_root_node = self.get_url_node_by_uuid(self.pages_root[node.pageref])
             if dev_debug:
                 self.logger.warning(f'Failed to attach URLNode in the normal process, attaching node to page {node.pageref} - Node: {page_root_node.uuid} - {page_root_node.name}.')
             self._make_subtree(page_root_node, [node])
-        elif self.url_tree.search_nodes(name=self.har.final_redirect):
-            # Generally, when we have a bunch of redirects, they do not branch out before the final landing page
-            # *but* it is not always the case: some intermediary redirects will have calls to 3rd party pages.
+        elif self.rendered_node != self.url_tree:
+            # Generally, when we have a bunch of redirects, they (generally) do not branch out
+            # before the final landing page *but* it is not always the case: some intermediary
+            # redirects will have calls to 3rd party pages.
             # Hopefully, this last case was taken care of in the branch above.
-            # In this branch, we get the landing page after the redirects (if any), and attach the node to it.
+            # In this branch, we get the landing page after the redirects, and attach the node to it.
+
+            # We skip this call if there are no redirects as it is the very last fallback at the
+            # end of this method anyway
             if dev_debug:
                 self.logger.warning(f'Failed to attach URLNode in the normal process, attaching node to final redirect: {self.har.final_redirect}.')
-            self._make_subtree(self.url_tree.search_nodes(name=self.har.final_redirect)[0], [node])
+            self._make_subtree(self.rendered_node, [node])
         elif 'pages' in self.har.har['log']:
             # No luck, the node is root for this pageref, let's attach it to the prior page in the list, or the very first node (tree root)
             page_before = self.har.har['log']['pages'][0]

har2tree/nodes.py

@@ -27,6 +27,7 @@ import json_stream  # type: ignore
 from bs4 import BeautifulSoup
 from ete3 import TreeNode  # type: ignore
 from publicsuffixlist import PublicSuffixList  # type: ignore
+from requests_toolbelt.multipart import decoder  # type: ignore
 from w3lib.html import strip_html5_whitespace
 from w3lib.url import canonicalize_url, safe_url_string
 
@@ -217,10 +218,19 @@ class URLNode(HarTreeNode):
             decoded_posted_data: list[Any] | str | bytes | int | float | bool | dict[str, str] | dict[str, list[str]] | None = None
             if 'postData' not in self.request or 'text' not in self.request['postData']:
                 self.logger.debug('POST request with no content.')
+                self.add_feature('posted_data_info', "No content.")
             elif not self.request['postData']['text']:
                 # If the POST content is empty
                 self.logger.debug('Empty POST request.')
                 decoded_posted_data = ''
+                self.add_feature('posted_data_info', "Empty request.")
+            elif self.request['postData']['text'].startswith('\x1f\uFFFD\x08'):
+                # b'\x1f\xef\xbf\xbd\x08', decoded to UTF-8
+                # => the replacement character
+                # https://www.cogsci.ed.ac.uk/~richard/utf-8.cgi?input=%EF%BF%BD&mode=char
+                self.logger.debug('Got a garbled gzipped POST blob.')
+                self.add_feature('posted_data_info', "It was a POSTed gzipped blob, but the data has been garbled.")
+                decoded_posted_data = self.request['postData']['text']
             elif self.request['postData'].get('params'):
                 # NOTE 2025-08-08
                 # if the posted data mimetype is "application/x-www-form-urlencoded"
@@ -231,10 +241,11 @@ class URLNode(HarTreeNode):
                 # TODO: some processing on the data part (it's often a json blob)
                 self.logger.debug('Got a params POST.')
                 decoded_posted_data = {entry['name']: entry['value'] for entry in self.request['postData']['params']}
+                self.add_feature('posted_data_info', "POST request as URL params.")
             else:
-                # NOTE 2023-08-22: Blind attempt to base64 decode the data
                 self.logger.debug('Got a normal POST')
                 try:
+                    # NOTE 2023-08-22: Blind attempt to base64 decode the data
                     decoded_posted_data = self._dirty_safe_b64decode(self.request['postData']['text'])
                 except binascii.Error:
                     decoded_posted_data = self.request['postData']['text']
@@ -254,12 +265,16 @@ class URLNode(HarTreeNode):
                                 decoded_posted_data = unquote_plus(decoded_posted_data)
                             if isinstance(decoded_posted_data, str):
                                 decoded_posted_data = parse_qs(decoded_posted_data)
+                            self.add_feature('posted_data_info', "Successfully decoded POST request.")
                         except Exception as e:
                             self.logger.warning(f'Unable to unquote or parse form data "{decoded_posted_data!r}": {e}')
+                            self.add_feature('posted_data_info', "Unable to decode POST request.")
                     elif (mimetype_lower.startswith('application/json')
                           or mimetype_lower.startswith('application/csp-report')
                           or mimetype_lower.startswith('application/x-amz-json-1.1')
                           or mimetype_lower.startswith('application/reports+json')
+                          or mimetype_lower.startswith('application/vnd.adobe.dc+json')
+                          or mimetype_lower.startswith('application/ion+json')
                           or mimetype_lower.endswith('json')
                           ):
                         if isinstance(decoded_posted_data, (str, bytes)):
@@ -267,7 +282,9 @@ class URLNode(HarTreeNode):
                             try:
                                 # NOTE 2023-08-22: loads here may give us a int, float or a bool.
                                 decoded_posted_data = json.loads(decoded_posted_data)
+                                self.add_feature('posted_data_info', "Successfully decoded POST request.")
                             except Exception:
+                                self.add_feature('posted_data_info', "Unable to decode POST request.")
                                 if isinstance(decoded_posted_data, (str, bytes)):
                                     self.logger.warning(f"Expected json, got garbage: {mimetype_lower} - {decoded_posted_data[:20]!r}[...]")
                                 else:
@@ -283,71 +300,109 @@ class URLNode(HarTreeNode):
                                 raise ValueError(f'Invalid type: {type(decoded_posted_data)}')
                             streamed_data = json_stream.load(to_stream)
                             decoded_posted_data = json_stream.to_standard_types(streamed_data)
+                            self.add_feature('posted_data_info', "Successfully decoded POST request.")
                         except Exception:
                             if isinstance(decoded_posted_data, (str, bytes)):
                                 self.logger.warning(f"Expected json stream, got garbage: {mimetype_lower} - {decoded_posted_data[:20]!r}[...]")
                             else:
                                 self.logger.warning(f"Expected json stream, got garbage: {mimetype_lower} - {decoded_posted_data}")
-                    elif mimetype_lower.startswith('multipart/form-data'):
-                        # FIXME multipart content (similar to email). Not totally sure what do do with it tight now.
-                        self.logger.debug(f'Got a POST {mimetype_lower}: {decoded_posted_data!r}')
-                        pass
+                            self.add_feature('posted_data_info', "Unable to decode POST request.")
+                    elif mimetype_lower.startswith('multipart'):
+                        self.add_feature('posted_data_info', f"Decoding {mimetype_lower} is partially supported.")
+                        if isinstance(decoded_posted_data, str):
+                            # must be encoded for decoding
+                            multipart_to_decode = decoded_posted_data.encode()
+                        elif isinstance(decoded_posted_data, bytes):
+                            multipart_to_decode = decoded_posted_data
+                        else:
+                            raise ValueError(f'Invalid type for multipart POST: {type(decoded_posted_data)}')
+                        if b"\r\n" not in multipart_to_decode:
+                            # the decoder wants that
+                            multipart_to_decode = multipart_to_decode.replace(b"\n", b"\r\n")
+                        try:
+                            multipart_data = decoder.MultipartDecoder(multipart_to_decode, mimetype_lower)
+                            decoded_posted_data = []
+                            for part in multipart_data.parts:
+                                headers = {k.decode(): v.decode() for k, v in part.headers.items()}
+                                content = part.text
+                                decoded_posted_data.append({'headers': headers, 'content': content})
+                        except Exception as e:
+                            self.logger.warning(f'Unable to decode multipart POST: {e}')
+                            self.add_feature('posted_data_info', "Unable to decode multipart in POST request.")
+
                     elif mimetype_lower.startswith('application/x-protobuf'):
                         # FIXME If possible, decode?
                         self.logger.debug(f'Got a POST {mimetype_lower}: {decoded_posted_data!r}')
-                        pass
+                        self.add_feature('posted_data_info', f"Decoding {mimetype_lower} is not supported yet.")
                     elif mimetype_lower.startswith('text') and isinstance(decoded_posted_data, (str, bytes)):
                         try:
                             # NOTE 2023-08-22: Quite a few text entries are in fact json, give it a shot.
                             # loads here may give us a int, float or a bool.
                             decoded_posted_data = json.loads(decoded_posted_data)
+                            self.add_feature('posted_data_info', "Decoded JSON out of POST request.")
                         except Exception:
                             # keep it as it is otherwise.
                             pass
                     elif mimetype_lower.endswith('javascript'):
                         # keep it as it is
                         self.logger.warning(f'Got a POST {mimetype_lower}: {decoded_posted_data!r}')
-                        pass
-                    elif mimetype_lower == '?':
-                        # Just skip it, no need to go in the warnings
+                        self.add_feature('posted_data_info', f"Pretty rendering of {mimetype_lower} is not supported yet.")
+                    elif mimetype_lower in ['?', '*/*']:
                         self.logger.warning(f'Got a POST {mimetype_lower}: {decoded_posted_data!r}')
-                        pass
+                        self.add_feature('posted_data_info', f"Weird MimeType ({mimetype_lower}) is not supported yet.")
                     elif mimetype_lower == 'application/binary':
-                        # generally a broken gzipped blob
-                        self.logger.debug(f'Got a POST {mimetype_lower}, most probably a broken gziped blob: {decoded_posted_data!r}')
+                        self.logger.warning(f'Got a POST {mimetype_lower}, not a broken gziped blob: {decoded_posted_data!r}')
+                        self.add_feature('posted_data_info', f"MimeType ({mimetype_lower}) is not supported yet.")
                     elif mimetype_lower in ['application/octet-stream']:
                         # Should flag it, maybe?
                         self.logger.warning(f'Got a POST {mimetype_lower}: {decoded_posted_data!r}')
-                        pass
+                        self.add_feature('posted_data_info', f"MimeType ({mimetype_lower}) is not supported yet.")
                     elif mimetype_lower in ['application/grpc-web+proto']:
                         # Can be decoded?
                         self.logger.warning(f'Got a POST {mimetype_lower} - can be decoded: {decoded_posted_data!r}')
+                        self.add_feature('posted_data_info', f"MimeType ({mimetype_lower}) is not supported yet.")
                     elif mimetype_lower in ['application/unknown']:
                         # Weird but already seen stuff
                         self.logger.warning(f'Got a POST {mimetype_lower}: {decoded_posted_data!r}')
-                        pass
+                        self.add_feature('posted_data_info', f"MimeType ({mimetype_lower}) is not supported yet.")
                     else:
-                        self.logger.warning(f'Unexpected mime type: {mimetype_lower}')
+                        self.logger.warning(f'Unexpected mime type: {mimetype_lower} - {decoded_posted_data!r}')
+                        self.add_feature('posted_data_info', f"Unexpected MimeType ({mimetype_lower}) is not supported yet.")
                 else:
                     self.logger.warning(f'Missing mimetype in POST: {self.request["postData"]}')
+                    self.add_feature('posted_data_info', "Missing MimeType, not sure what to do.")
 
             # NOTE 2023-08-22: Blind attempt to process the data as json
-            if isinstance(decoded_posted_data, (str, bytes)):
+            if decoded_posted_data and isinstance(decoded_posted_data, (str, bytes)):
                 try:
                     decoded_posted_data = json.loads(decoded_posted_data)
                 except Exception:
                     pass
 
-            if isinstance(decoded_posted_data, bytes):
+            if decoded_posted_data and isinstance(decoded_posted_data, bytes):
                 # NOTE 2023-08-22: Blind attempt to decode the bytes
                 # Try to decode it as utf-8
                 try:
                     decoded_posted_data = decoded_posted_data.decode('utf-8')
                 except Exception:
                     pass
+
             self.add_feature('posted_data', decoded_posted_data)
             if 'postData' in self.request and self.request['postData'].get('mimeType'):
                 self.add_feature('posted_data_mimetype', self.request['postData']['mimeType'])
+            # Get size, post decode.
+            if not decoded_posted_data:
+                # empty or None, set to 0
+                self.add_feature('posted_data_size', 0)
+            elif isinstance(decoded_posted_data, (list, dict)):
+                # set size to the json dump
+                self.add_feature('posted_data_size', len(json.dumps(decoded_posted_data)))
+            elif isinstance(decoded_posted_data, (str, bytes)):
+                # length
+                self.add_feature('posted_data_size', len(decoded_posted_data))
+            else:
+                # Stringify and len
+                self.add_feature('posted_data_size', len(str(decoded_posted_data)))
 
         self.add_feature('response', har_entry['response'])
         try:
@@ -531,7 +586,7 @@ class URLNode(HarTreeNode):
             return href
 
         if not hasattr(self, 'rendered_html') or not self.rendered_html:
-            raise Har2TreeError('Not the node of a page rendered, invalid request.')
+            raise Har2TreeError('Not the node of a page rendered ({self.uuid}), invalid request.')
         urls: set[str] = set()
 
         # The simple ones: the links.

{har2tree-1.31.5.dist-info → har2tree-1.32.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: har2tree
-Version: 1.31.5
+Version: 1.32.0
 Summary: HTTP Archive (HAR) to ETE Toolkit generator
 License: BSD-3-Clause
 Author: Raphaël Vinot
@@ -21,7 +21,7 @@ Classifier: Topic :: Internet
 Classifier: Topic :: Security
 Provides-Extra: docs
 Requires-Dist: Sphinx (>=8.2.3) ; (python_version >= "3.11") and (extra == "docs")
-Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.13.4)
+Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.13.5)
 Requires-Dist: ete3 (>=3.1.3)
 Requires-Dist: filetype (>=1.2.0)
 Requires-Dist: json-stream (>=2.3.3,<3.0.0)
@@ -30,7 +30,8 @@ Requires-Dist: multipart (>=1.3.0,<2.0.0)
 Requires-Dist: numpy (<2.1) ; python_version < "3.10"
 Requires-Dist: numpy (<2.3) ; python_version < "3.11"
 Requires-Dist: numpy (>=2.3.2) ; python_version >= "3.11"
-Requires-Dist: publicsuffixlist (>=1.0.2.20250809)
+Requires-Dist: publicsuffixlist (>=1.0.2.20250824)
+Requires-Dist: requests-toolbelt (>=1.0.0,<2.0.0)
 Requires-Dist: six (>=1.17.0) ; extra == "docs"
 Requires-Dist: tinycss2 (>=1.4.0)
 Requires-Dist: w3lib (>=2.3.1)

har2tree-1.32.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+har2tree/__init__.py,sha256=Na3mxHkUBq3rzYbxiLNJF37DxH5mcghSorjzXw5Teug,422
+har2tree/har2tree.py,sha256=PBRJZk-cqIOctbrIav4v5z2wKUFApayl4SQmLTKdF6E,44438
+har2tree/helper.py,sha256=CgeXqfBeHs8SbkW7TRNKqJBTZLAu63KggQjbGHCZAGI,20681
+har2tree/nodes.py,sha256=8Z1CTCQvLF6TQIYK7UO0BzXsJMIwwZ6uVGPwa1Kv7zo,37516
+har2tree/parser.py,sha256=4yej1OcVYAIiLfzYZsO9WCw3WyM_ykDTuvpW7UO1ROE,3645
+har2tree/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+har2tree-1.32.0.dist-info/LICENSE,sha256=Xa4EVROgJsEo10CW-ISCRiw0TtqdKz1JuM3BBLBM55c,1803
+har2tree-1.32.0.dist-info/METADATA,sha256=gV4DHjc7BnpGnEQlxqLl0bYHK3xPgjsVNUqeqkgRE58,2253
+har2tree-1.32.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+har2tree-1.32.0.dist-info/RECORD,,

har2tree-1.31.5.dist-info/RECORD

@@ -1,10 +0,0 @@
-har2tree/__init__.py,sha256=Na3mxHkUBq3rzYbxiLNJF37DxH5mcghSorjzXw5Teug,422
-har2tree/har2tree.py,sha256=47x9X5tY69f9SXkYJgJsnAaX2kxgXHgzFThGz6M86Zw,44495
-har2tree/helper.py,sha256=CgeXqfBeHs8SbkW7TRNKqJBTZLAu63KggQjbGHCZAGI,20681
-har2tree/nodes.py,sha256=a-5tk_AbnIklbdujlesb_1E0KGnSyK0OsTnbnd5i0D4,32961
-har2tree/parser.py,sha256=4yej1OcVYAIiLfzYZsO9WCw3WyM_ykDTuvpW7UO1ROE,3645
-har2tree/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-har2tree-1.31.5.dist-info/LICENSE,sha256=Xa4EVROgJsEo10CW-ISCRiw0TtqdKz1JuM3BBLBM55c,1803
-har2tree-1.31.5.dist-info/METADATA,sha256=5QfFL4ESUuWJn7JuxcnLSgw70q3MGZoDfe9PJFS5JkA,2203
-har2tree-1.31.5.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-har2tree-1.31.5.dist-info/RECORD,,