xbase-util 0.5.0__tar.gz → 0.5.2__tar.gz

{xbase_util-0.5.0 → xbase_util-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase_util
-Version: 0.5.0
+Version: 0.5.2
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt

{xbase_util-0.5.0 → xbase_util-0.5.2}/setup.py

@@ -3,7 +3,7 @@ from distutils.core import setup
 from setuptools import find_packages
 
 setup(name="xbase_util",
-      version="0.5.0",
+      version="0.5.2",
       description="网络安全基础工具",
       long_description="包含提取，预测，训练的基础工具",
       author="xyt",

{xbase_util-0.5.0 → xbase_util-0.5.2}/xbase_util/packet_util.py

@@ -1,6 +1,6 @@
 import re
 
-from scapy.layers.inet import TCP
+from scapy.layers.inet import TCP, IP
 from scapy.packet import Raw
 
 from xbase_util.xbase_constant import plain_content_type_columns, packetKeyname, src_dst_header, statisticHeader, \
@@ -58,85 +58,67 @@ def get_all_columns(
     return result_columns
 
 
+req_pattern = re.compile(r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?\r\n\r\n",
+                         re.DOTALL)
+res_pattern = re.compile(r"HTTP/\d\.\d \d{3}.*", re.DOTALL)
+req_body_pattern = re.compile(
+    r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?(?=HTTP/\d\.\d)", re.DOTALL)
+
+
 def get_all_packets_by_reg(packets):
-    req_pattern = re.compile(r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?\r\n\r\n",re.DOTALL)
-    req_body_pattern = re.compile(
-        r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?(?=HTTP/\d\.\d)",re.DOTALL)
-    res_pattern = re.compile(r"HTTP/\d\.\d \d{3}.*",re.DOTALL)
-    tcp_packet_map = {}
-    for packet in packets:
-        if packet.haslayer(TCP) and packet.haslayer(Raw):
-            raw_data = bytes(packet[Raw].load)
-            ack = f"{packet[TCP].ack}"
-            seq = packet[TCP].seq
-            time = packet[TCP].time
-            if f"{packet[TCP].ack}" not in tcp_packet_map:
-                tcp_packet_map[ack] = {
-                    "data": raw_data,
-                    "time": [time],
-                    "seq": [seq],
-                    "last_seq": seq,
-                    "ack": ack,
-                    "len": [len(raw_data)],
-                    "last_len": len(raw_data)
-                }
-            else:
-                tcp_packet_map[ack]['data'] += raw_data
-                tcp_packet_map[ack]['time'].append(time)
-                tcp_packet_map[ack]['seq'].append(seq)
-                tcp_packet_map[ack]['last_len'] = len(raw_data)
-                tcp_packet_map[ack]['last_seq'] = seq
+    packets = [packet for packet in packets if packet.haslayer(TCP) and packet.haslayer(IP) and packet.haslayer(Raw)]
     packet_list = []
-    for ack, data_set in tcp_packet_map.items():
-        data_str = data_set['data'].decode("utf-8", errors="ignore")
-        request_re = re.search(req_pattern, data_str)
-        if request_re is None:
-            continue
-        next_ack = f"{data_set['last_len'] + data_set['last_seq']}"
-        packet_data = data_set['data']
-        request_time = data_set['time']
-        response_time=[]
-        req_len = len(packet_data)
-        res_len = 0
-        while True:
-            # 持续往下一个包找，直到下一个包是请求为止，因为下一个包可能还是属于这个包的一部分，也可能是响应的一部分
-            # 下一个包的ack存在
-            if next_ack not in tcp_packet_map:
-                print("没找到新的ack")
-                break
-            new_packet = tcp_packet_map[next_ack]
-            # 判断新的包是不是响应包
-            res_match = re.search(res_pattern, filter_visible_chars(new_packet['data']))
-            if res_match is None:
-                req_len += len(new_packet['data'])
-                request_time += new_packet['time']
-            else:
-                print("匹配到响应")
-                res_len += len(new_packet['data'])
-                response_time += new_packet['time']
-
-            # 判断新的包是不是第二个请求包
-            if re.search(req_pattern, new_packet['data'].decode("utf-8", errors="ignore")):
-                print("这个包是个新的请求包的开头，停止查找")
-                break
-            packet_data += new_packet['data']
-            next_ack = f"{new_packet['last_len'] + new_packet['last_seq']}"
-        map = {}
-        data = filter_visible_chars(packet_data)
-        match_req = re.search(
-            req_body_pattern,
-            data)
-        match_res = re.search(res_pattern, data)
-        map['data'] = packet_data
-        map['req_len'] = req_len
-        map['res_len'] = res_len
-        map['request_time'] = request_time
-        map['response_time'] = response_time
-        map['req'] = match_req.group() if match_req is not None else ""
-        map['res'] = match_res.group() if match_res is not None else ""
-        packet_list.append(map)
+    my_map = {
+        'req_data': b'',
+        'res_data': b'',
+        'req_text': '',
+        'res_text': '',
+        'req_time': [],
+        'res_time': []
+    }
+    last_is_req = None
+    for item in packets:
+        data = item[Raw].load
+        time = float(item.time)
+        req_match = req_pattern.search(filter_visible_chars(data))
+        res_match = res_pattern.search(filter_visible_chars(data))
+        if req_match is not None or res_match is not None:
+            if req_match:
+                # 新的请求：请求时间不为空或者响应时间不为空，说明不为空，添加到列表并清空数据
+                if len(my_map['req_time']) != 0 or len(my_map['res_time']) != 0:
+                    packet_list.append(my_map.copy())
+                my_map = {
+                    'req_data': data,
+                    'res_data': b'',
+                    'req_text': filter_visible_chars(data),
+                    'res_text': '',
+                    'req_time': [time],
+                    'res_time': []
+                }
+                last_is_req = True
+            if res_match:
+                my_map['res_data'] += data
+                my_map['res_text'] = filter_visible_chars(my_map['res_data'])
+                my_map['res_time'].append(time)
+                last_is_req = False
+        else:
+            # 不是请求不是相应，就是中间的包
+            if last_is_req is None:
+                # 一开始就没匹配到请求或者响应头，那就不管即使是中间的包
+                continue
+            if last_is_req is True:
+                my_map['req_time'].append(time)
+                my_map['req_data'] += data
+                my_map['req_text'] = filter_visible_chars(my_map['req_data'])
+            elif last_is_req is False:
+                my_map['res_time'].append(time)
+                my_map['res_data'] += data
+                my_map['res_text'] = filter_visible_chars(my_map['res_data'])
+    if len(my_map['req_time']) != 0 or len(my_map['res_time']) != 0:
+        packet_list.append(my_map.copy())
     return packet_list
 
+
 def get_body(param):
     body = "".join([item.strip() for item in param.split("\r\n\r\n") if item.strip() != "" and "HTTP/" not in param])
     return "" if body is None else body
@@ -159,8 +141,8 @@ def get_detail_by_package(packets_from_pcap, publicField, use_regx):
     """
     res_field = publicField.copy()
     if use_regx:
-        req = packets_from_pcap['req']
-        res = packets_from_pcap['res']
+        req = packets_from_pcap['req_text']
+        res = packets_from_pcap['res_text']
     else:
         res = packets_from_pcap["response"]
         req = packets_from_pcap["request"]

{xbase_util-0.5.0 → xbase_util-0.5.2}/xbase_util.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase-util
-Version: 0.5.0
+Version: 0.5.2
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt