xbase-util 0.7.2__tar.gz → 0.7.4__tar.gz

{xbase_util-0.7.2 → xbase_util-0.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase_util
-Version: 0.7.2
+Version: 0.7.4
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt

{xbase_util-0.7.2 → xbase_util-0.7.4}/setup.py

@@ -3,7 +3,7 @@ from distutils.core import setup
 from setuptools import find_packages
 
 setup(name="xbase_util",
-      version="0.7.2",
+      version="0.7.4",
       description="网络安全基础工具",
       long_description="包含提取，预测，训练的基础工具",
       author="xyt",

{xbase_util-0.7.2 → xbase_util-0.7.4}/xbase_util/packet_util.py

@@ -70,37 +70,6 @@ http_req_method = re.compile(r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]*
 http_req_path = re.compile(r"(?:GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH)\s+(\/[^\s]*)\s+HTTP\/\d\.\d")
 res_status_code_pattern = re.compile(r"HTTP\/\d\.\d\s+(\d{3})\s+.*")
 
-
-# def get_all_packets_by_reg(packets):
-#     http_Req_Raw = {}
-#     http_methods = ("POST /", "PUT /", "OPTIONS /", "DELETE /", "GET /")
-#     for packet in packets:
-#         if TCP in packet and Raw in packet:
-#             data = packet[Raw].load
-#             ack = packet[TCP].ack
-#             next_ack = packet[TCP].seq + len(data)
-#             if ack not in http_Req_Raw:
-#                 http_Req_Raw[ack] = {"time": [float(packet.time)], "data": data, "next_ack": next_ack}
-#             else:
-#                 http_Req_Raw[ack]["time"].append(float(packet.time))
-#                 http_Req_Raw[ack]["data"] += data
-#                 http_Req_Raw[ack]["next_ack"] = next_ack
-#     packet_list = [
-#         {
-#             'req_data': item['data'],
-#             'res_data': http_Req_Raw[item['next_ack']]['data'],
-#             'req_text': filter_visible_chars(item['data']),
-#             'res_text': filter_visible_chars(http_Req_Raw[item['next_ack']]['data']),
-#             'req_time': item['time'],
-#             'res_time': http_Req_Raw[item['next_ack']]['time']
-#         }
-#         for ack, item in http_Req_Raw.items()
-#         if item['next_ack'] in http_Req_Raw  # 检查 next_ack 是否存在
-#            and any(method in filter_visible_chars(item['data']) for method in http_methods)
-#     ]
-#     return packet_list
-
-
 def get_header_value(header_set, value):
     result = [item for item in header_set if value in item]
     if len(result) != 0:
@@ -139,9 +108,9 @@ def get_detail_by_package(publicField, req_header, req_body, res_header, res_bod
     res_field['plain_body_src'] = ""
     res_field['plain_body_dst'] = ""
     if content_type_is_plain(req_header):
-        res_field['plain_body_src'] = f"{req_body}"
+        res_field['plain_body_src'] = req_body
     if content_type_is_plain(res_header):
-        res_field['plain_body_dst'] = f"{res_body}"
+        res_field['plain_body_dst'] = res_body
 
     http_server_version_res = http_version.findall(res_header)
     res_field['http.serverVersion'] = http_server_version_res[0] if len(http_server_version_res) > 0 else ""

{xbase_util-0.7.2 → xbase_util-0.7.4}/xbase_util/segment.py

@@ -1,10 +1,15 @@
 import copy
+import gzip
 import re
+import zlib
+from io import BytesIO
 
 import numpy as np
 from scapy.all import *
 from scapy.layers.inet import TCP
 
+from xbase_util.packet_util import filter_visible_chars
+
 REQUEST_LINE_RE = re.compile(rb"^(GET|POST|PUT|DELETE|OPTIONS|HEAD|PATCH)\s[^\r\n]+\r\n", re.MULTILINE)
 RESPONSE_LINE_RE = re.compile(rb"^HTTP/\d\.\d\s+\d{3}\s?[^\r\n]*", re.IGNORECASE)
 
@@ -50,7 +55,8 @@ def read_packets(packets):
         tmp_packets.clear()
     return packet_list
 
-
+pattern_chuncked = re.compile(rb"Transfer-Encoding:\s*chunked", re.IGNORECASE)
+pattern_gzip = re.compile(rb"Content-Encoding:\s*gzip", re.IGNORECASE)
 def parse_req_or_res(data, pkts):
     if data.find(b"\r\n\r\n") != -1:
         res = data.split(b"\r\n\r\n", 1)
@@ -59,8 +65,6 @@ def parse_req_or_res(data, pkts):
     else:
         header = data
         body = b''
-    pattern_chuncked = re.compile(rb"Transfer-Encoding:\s*chunked", re.IGNORECASE)
-    pattern_gzip = re.compile(rb"Content-Encoding:\s*gzip", re.IGNORECASE)
     chuncked_pattern = pattern_chuncked.search(header)
     gzip_pattern = pattern_gzip.search(header)
     if chuncked_pattern and b'chunked' in chuncked_pattern.group():
@@ -99,15 +103,16 @@ def parse_req_or_res(data, pkts):
             result_body = body
     if gzip_pattern and b'gzip' in gzip_pattern.group():
         try:
+
             decompressed = gzip.decompress(result_body)
             result_body_str = "\n".join(
-                [line.strip() for line in decompressed.decode("utf-8", errors="replace").splitlines() if
+                [line.strip() for line in filter_visible_chars(decompressed).splitlines() if
                  line.strip() != ""])
         except Exception as e:
-            result_body_str = result_body.decode("utf-8", errors="replace")
+            result_body_str = filter_visible_chars(result_body)
     else:
-        result_body_str = result_body.decode("utf-8", errors="replace")
-    return header.decode("utf-8", errors="replace"), result_body_str, [float(pkt.time) for pkt in pkts]
+        result_body_str = filter_visible_chars(result_body)
+    return filter_visible_chars(header), result_body_str, [float(pkt.time) for pkt in pkts]
 
 
 def get_all_packets_by_segment(packets):
@@ -126,6 +131,8 @@ def get_all_packets_by_segment(packets):
         response = [item for item in response_packets if item['first_seq'] == ack]
         # print(f"找到对应的响应：{len(response)}")
         # print(f"请求：{request['data'].decode('utf-8', errors='replace')}")
+        # gzip.decompress(response[0]['data'].split(b'\r\n\r\n')[1])
+        # decompressed_data = zlib.decompress((response[0]['data'].split(b'\r\n\r\n')[1])[10:], wbits=zlib.MAX_WBITS | 16)
         if len(response) > 0:
             res_header, res_body, res_times = parse_req_or_res(response[0]['data'], response[0]['pkts'])
             req_header, req_body, req_times = parse_req_or_res(request['data'], request['pkts'])

{xbase_util-0.7.2 → xbase_util-0.7.4}/xbase_util.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase-util
-Version: 0.7.2
+Version: 0.7.4
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt