xbase-util 0.4.6__tar.gz → 0.4.8__tar.gz

{xbase_util-0.4.6 → xbase_util-0.4.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase_util
-Version: 0.4.6
+Version: 0.4.8
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt

{xbase_util-0.4.6 → xbase_util-0.4.8}/setup.py

@@ -3,7 +3,7 @@ from distutils.core import setup
 from setuptools import find_packages
 
 setup(name="xbase_util",
-      version="0.4.6",
+      version="0.4.8",
       description="网络安全基础工具",
       long_description="包含提取，预测，训练的基础工具",
       author="xyt",

{xbase_util-0.4.6 → xbase_util-0.4.8}/xbase_util/packet_util.py

@@ -1,6 +1,7 @@
 import re
 
 from scapy.layers.inet import TCP
+from scapy.packet import Raw
 
 from xbase_util.xbase_constant import plain_content_type_columns, packetKeyname, src_dst_header, statisticHeader, \
     features_key, plain_body_columns
@@ -57,45 +58,87 @@ def get_all_columns(
     return result_columns
 
 
-def get_all_packets_by_regx(packets):
-    """
-    通过正则pcap获取所有包的数据
-    :param packets:
-    :return:
-    """
-    streams = b""
-    for pkt in packets:
-        if TCP in pkt:
-            streams += bytes(pkt[TCP].payload)
-    text = filter_visible_chars(streams)
-    pattern = r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d"
-    requests = re.split(f"(?={pattern})", text, re.M)
-    all_packets = []
-    for item in requests:
-        if len(re.findall(pattern, item)) != 0:
-            request_text = ""
-            response_text = ""
-            response_text_list = re.findall(r"HTTP/\d\.\d \d{3}[\s\S]*", item)
-            if len(response_text_list) != 0:
-                # 有响应数据
-                response_text = response_text_list[0]
-            if response_text == "":
-                # 没有响应数据，那么全是请求数据
-                request_text = item
+def get_all_packets_by_reg(packets):
+    req_pattern = re.compile(r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?\r\n\r\n",
+                             re.DOTALL)
+    req_res_pattern = re.compile(
+        r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?(?=HTTP/\d\.\d \d{3} [a-zA-Z]+|$)",
+        re.DOTALL)
+    res_pattern = re.compile(r"HTTP/\d\.\d \d{3} [a-zA-Z]+.*", re.DOTALL)
+    tcp_packet_map = {}
+    for packet in packets:
+        if packet.haslayer(TCP) and packet.haslayer(Raw):
+            raw_data = bytes(packet[Raw].load)
+            ack = f"{packet[TCP].ack}"
+            seq = packet[TCP].seq
+            time = packet[TCP].time
+            if f"{packet[TCP].ack}" not in tcp_packet_map:
+                tcp_packet_map[ack] = {
+                    "data": raw_data,
+                    "time": [time],
+                    "seq": [seq],
+                    "last_seq": seq,
+                    "ack": ack,
+                    "len": [len(raw_data)],
+                    "last_len": len(raw_data)
+                }
+            else:
+                tcp_packet_map[ack]['data'] += raw_data
+                tcp_packet_map[ack]['time'].append(time)
+                tcp_packet_map[ack]['seq'].append(seq)
+                tcp_packet_map[ack]['last_len'] = len(raw_data)
+                tcp_packet_map[ack]['last_seq'] = seq
+    packet_list = []
+    for ack, data_set in tcp_packet_map.items():
+        data_str = data_set['data'].decode("utf-8", errors="ignore")
+        request_re = re.search(req_pattern, data_str)
+        if request_re is None:
+            continue
+        next_ack = f"{data_set['last_len'] + data_set['last_seq']}"
+        packet_data = data_set['data']
+        request_time = data_set['time']
+        req_len = len(packet_data)
+        res_len = 0
+        while True:
+            # 持续往下一个包找，直到下一个包是请求为止，因为下一个包可能还是属于这个包的一部分，也可能是响应的一部分
+            # 下一个包的ack存在
+            if next_ack not in tcp_packet_map:
+                print("没找到新的ack")
+                break
+            new_packet = tcp_packet_map[next_ack]
+            # 判断新的包是不是响应包
+
+            res_match = re.search(res_pattern, filter_visible_chars(new_packet['data']))
+            if res_match is None:
+                req_len += len(new_packet['data'])
             else:
-                # 有响应数据，用正则获取请求数据
-                request_re = re.search(
-                    r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?\r\n\r\n", item)
-                if request_re:
-                    request_text = request_re.group(0)
-                else:
-                    request_text = ""
-            all_packets.append({"req": request_text, "res": response_text})
-    return all_packets
-
-
-def get_body(param, is_src):
-    body = param.split("\r\n\r\n")[1].strip()
+                print("匹配到响应")
+                res_len += len(new_packet['data'])
+            # 判断新的包是不是第二个请求包
+            if re.search(req_pattern, new_packet['data'].decode("utf-8", errors="ignore")):
+                print("这个包是个新的请求包的开头，停止查找")
+                break
+            packet_data += new_packet['data']
+            request_time += new_packet['time']
+            next_ack = f"{new_packet['last_len'] + new_packet['last_seq']}"
+        map = {}
+        data = filter_visible_chars(packet_data)
+        match_req = re.search(
+            req_res_pattern,
+            data)
+        match_res = re.search(res_pattern, data)
+        map['data'] = packet_data
+        map['req_len'] = req_len
+        map['res_len'] = res_len
+        map['time'] = request_time
+        map['req'] = match_req.group() if match_req is not None else ""
+        map['res'] = match_res.group() if match_res is not None else ""
+        packet_list.append(map)
+    return packet_list
+
+
+def get_body(param):
+    body = "".join([item.strip() for item in param.split("\r\n\r\n") if item.strip() != "" and "HTTP/" not in param])
     return "" if body is None else body
 
 
@@ -143,9 +186,9 @@ def get_detail_by_package(packets_from_pcap, publicField, use_regx):
         res_field['plain_body_src'] = ""
         res_field['plain_body_dst'] = ""
         if content_type_is_plain(req):
-            res_field['plain_body_src'] = get_body(req, is_src=True)
+            res_field['plain_body_src'] = get_body(req)
         if content_type_is_plain(res):
-            res_field['plain_body_dst'] = get_body(res, is_src=False)
+            res_field['plain_body_dst'] = get_body(res)
 
     response_lines = res.strip().split("\n")
     http_response_lines = [item for item in response_lines if "HTTP" in item]

{xbase_util-0.4.6 → xbase_util-0.4.8}/xbase_util.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase-util
-Version: 0.4.6
+Version: 0.4.8
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt