har2tree 1.31.4__py3-none-any.whl → 1.31.5__py3-none-any.whl

har2tree/nodes.py

@@ -15,13 +15,15 @@ from base64 import b64decode
 from datetime import datetime, timedelta
 from functools import lru_cache, cached_property
 from hashlib import sha256
-from io import BytesIO
+from io import BytesIO, StringIO
 from pathlib import Path
 from typing import Any
 from collections.abc import MutableMapping
-from urllib.parse import unquote_plus, urlparse, urljoin
+from urllib.parse import unquote_plus, urlparse, urljoin, parse_qs
 
 import filetype  # type: ignore
+import json_stream  # type: ignore
+
 from bs4 import BeautifulSoup
 from ete3 import TreeNode  # type: ignore
 from publicsuffixlist import PublicSuffixList  # type: ignore
@@ -212,7 +214,7 @@ class URLNode(HarTreeNode):
             self.add_feature('user_agent', '')
 
         if 'method' in self.request and self.request['method'] == 'POST':
-            decoded_posted_data: str | bytes | int | float | bool | dict[str, str] | None = None
+            decoded_posted_data: list[Any] | str | bytes | int | float | bool | dict[str, str] | dict[str, list[str]] | None = None
             if 'postData' not in self.request or 'text' not in self.request['postData']:
                 self.logger.debug('POST request with no content.')
             elif not self.request['postData']['text']:
@@ -243,19 +245,20 @@ class URLNode(HarTreeNode):
                         # NOTE: this should never happen as there should
                         # be something in self.request['postData']['params']
                         # and we already processed it before but just in case...
-                        self.logger.warning(f'Got a application/x-www-form-urlencoded without params key: {self.request}')
+                        self.logger.debug('Got a application/x-www-form-urlencoded without params key')
                         # 100% sure there will be websites where decode will fail
                         try:
                             if isinstance(decoded_posted_data, bytes):
                                 decoded_posted_data = decoded_posted_data.decode()
                             if isinstance(decoded_posted_data, str):
                                 decoded_posted_data = unquote_plus(decoded_posted_data)
+                            if isinstance(decoded_posted_data, str):
+                                decoded_posted_data = parse_qs(decoded_posted_data)
                         except Exception as e:
-                            self.logger.warning(f'Unable to unquote form data "{decoded_posted_data!r}": {e}')
+                            self.logger.warning(f'Unable to unquote or parse form data "{decoded_posted_data!r}": {e}')
                     elif (mimetype_lower.startswith('application/json')
                           or mimetype_lower.startswith('application/csp-report')
                           or mimetype_lower.startswith('application/x-amz-json-1.1')
-                          or mimetype_lower.startswith('application/x-json-stream')
                           or mimetype_lower.startswith('application/reports+json')
                           or mimetype_lower.endswith('json')
                           ):
@@ -269,7 +272,22 @@ class URLNode(HarTreeNode):
                                     self.logger.warning(f"Expected json, got garbage: {mimetype_lower} - {decoded_posted_data[:20]!r}[...]")
                                 else:
                                     self.logger.warning(f"Expected json, got garbage: {mimetype_lower} - {decoded_posted_data}")
-
+                    elif mimetype_lower.startswith('application/x-json-stream'):
+                        try:
+                            to_stream: StringIO | BytesIO
+                            if isinstance(decoded_posted_data, str):
+                                to_stream = StringIO(decoded_posted_data)
+                            elif isinstance(decoded_posted_data, bytes):
+                                to_stream = BytesIO(decoded_posted_data)
+                            else:
+                                raise ValueError(f'Invalid type: {type(decoded_posted_data)}')
+                            streamed_data = json_stream.load(to_stream)
+                            decoded_posted_data = json_stream.to_standard_types(streamed_data)
+                        except Exception:
+                            if isinstance(decoded_posted_data, (str, bytes)):
+                                self.logger.warning(f"Expected json stream, got garbage: {mimetype_lower} - {decoded_posted_data[:20]!r}[...]")
+                            else:
+                                self.logger.warning(f"Expected json stream, got garbage: {mimetype_lower} - {decoded_posted_data}")
                     elif mimetype_lower.startswith('multipart/form-data'):
                         # FIXME multipart content (similar to email). Not totally sure what do do with it tight now.
                         self.logger.debug(f'Got a POST {mimetype_lower}: {decoded_posted_data!r}')
@@ -294,11 +312,17 @@ class URLNode(HarTreeNode):
                         # Just skip it, no need to go in the warnings
                         self.logger.warning(f'Got a POST {mimetype_lower}: {decoded_posted_data!r}')
                         pass
-                    elif mimetype_lower in ['application/octet-stream', 'application/binary']:
+                    elif mimetype_lower == 'application/binary':
+                        # generally a broken gzipped blob
+                        self.logger.debug(f'Got a POST {mimetype_lower}, most probably a broken gziped blob: {decoded_posted_data!r}')
+                    elif mimetype_lower in ['application/octet-stream']:
                         # Should flag it, maybe?
                         self.logger.warning(f'Got a POST {mimetype_lower}: {decoded_posted_data!r}')
                         pass
-                    elif mimetype_lower in ['application/unknown', 'application/grpc-web+proto']:
+                    elif mimetype_lower in ['application/grpc-web+proto']:
+                        # Can be decoded?
+                        self.logger.warning(f'Got a POST {mimetype_lower} - can be decoded: {decoded_posted_data!r}')
+                    elif mimetype_lower in ['application/unknown']:
                         # Weird but already seen stuff
                         self.logger.warning(f'Got a POST {mimetype_lower}: {decoded_posted_data!r}')
                         pass
@@ -322,6 +346,8 @@ class URLNode(HarTreeNode):
                 except Exception:
                     pass
             self.add_feature('posted_data', decoded_posted_data)
+            if 'postData' in self.request and self.request['postData'].get('mimeType'):
+                self.add_feature('posted_data_mimetype', self.request['postData']['mimeType'])
 
         self.add_feature('response', har_entry['response'])
         try:

{har2tree-1.31.4.dist-info → har2tree-1.31.5.dist-info}/METADATA

@@ -1,11 +1,11 @@
 Metadata-Version: 2.3
 Name: har2tree
-Version: 1.31.4
+Version: 1.31.5
 Summary: HTTP Archive (HAR) to ETE Toolkit generator
 License: BSD-3-Clause
 Author: Raphaël Vinot
 Author-email: raphael.vinot@circl.lu
-Requires-Python: >=3.9
+Requires-Python: >=3.9,<4.0
 Classifier: Intended Audience :: Information Technology
 Classifier: Intended Audience :: Science/Research
 Classifier: Intended Audience :: Telecommunications Industry
@@ -24,12 +24,13 @@ Requires-Dist: Sphinx (>=8.2.3) ; (python_version >= "3.11") and (extra == "docs
 Requires-Dist: beautifulsoup4[charset-normalizer,lxml] (>=4.13.4)
 Requires-Dist: ete3 (>=3.1.3)
 Requires-Dist: filetype (>=1.2.0)
+Requires-Dist: json-stream (>=2.3.3,<3.0.0)
 Requires-Dist: legacy-cgi (>=2.6.3) ; python_version >= "3.13,<4.0"
 Requires-Dist: multipart (>=1.3.0,<2.0.0)
 Requires-Dist: numpy (<2.1) ; python_version < "3.10"
 Requires-Dist: numpy (<2.3) ; python_version < "3.11"
 Requires-Dist: numpy (>=2.3.2) ; python_version >= "3.11"
-Requires-Dist: publicsuffixlist (>=1.0.2.20250802)
+Requires-Dist: publicsuffixlist (>=1.0.2.20250809)
 Requires-Dist: six (>=1.17.0) ; extra == "docs"
 Requires-Dist: tinycss2 (>=1.4.0)
 Requires-Dist: w3lib (>=2.3.1)

{har2tree-1.31.4.dist-info → har2tree-1.31.5.dist-info}/RECORD

@@ -1,10 +1,10 @@
 har2tree/__init__.py,sha256=Na3mxHkUBq3rzYbxiLNJF37DxH5mcghSorjzXw5Teug,422
 har2tree/har2tree.py,sha256=47x9X5tY69f9SXkYJgJsnAaX2kxgXHgzFThGz6M86Zw,44495
 har2tree/helper.py,sha256=CgeXqfBeHs8SbkW7TRNKqJBTZLAu63KggQjbGHCZAGI,20681
-har2tree/nodes.py,sha256=LfnVP66dhGNcbcoaao32WzzYGSkjOPqrwHE9Swh3sug,31005
+har2tree/nodes.py,sha256=a-5tk_AbnIklbdujlesb_1E0KGnSyK0OsTnbnd5i0D4,32961
 har2tree/parser.py,sha256=4yej1OcVYAIiLfzYZsO9WCw3WyM_ykDTuvpW7UO1ROE,3645
 har2tree/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-har2tree-1.31.4.dist-info/LICENSE,sha256=Xa4EVROgJsEo10CW-ISCRiw0TtqdKz1JuM3BBLBM55c,1803
-har2tree-1.31.4.dist-info/METADATA,sha256=axJorvFoHABeSnKQhWEn2oVRBtD2ghtaY7mSSiRM1Q8,2154
-har2tree-1.31.4.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-har2tree-1.31.4.dist-info/RECORD,,
+har2tree-1.31.5.dist-info/LICENSE,sha256=Xa4EVROgJsEo10CW-ISCRiw0TtqdKz1JuM3BBLBM55c,1803
+har2tree-1.31.5.dist-info/METADATA,sha256=5QfFL4ESUuWJn7JuxcnLSgw70q3MGZoDfe9PJFS5JkA,2203
+har2tree-1.31.5.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+har2tree-1.31.5.dist-info/RECORD,,