xbase-util 0.6.5__tar.gz → 0.6.7__tar.gz

{xbase_util-0.6.5 → xbase_util-0.6.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase_util
-Version: 0.6.5
+Version: 0.6.7
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt

{xbase_util-0.6.5 → xbase_util-0.6.7}/setup.py

@@ -3,7 +3,7 @@ from distutils.core import setup
 from setuptools import find_packages
 
 setup(name="xbase_util",
-      version="0.6.5",
+      version="0.6.7",
       description="网络安全基础工具",
       long_description="包含提取，预测，训练的基础工具",
       author="xyt",

{xbase_util-0.6.5 → xbase_util-0.6.7}/xbase_util/packet_util.py

@@ -1,7 +1,7 @@
+import copy
 import re
-import traceback
 
-from scapy.layers.inet import TCP, IP
+from scapy.layers.inet import TCP
 from scapy.packet import Raw
 
 from xbase_util.xbase_constant import plain_content_type_columns, packetKeyname, src_dst_header, statisticHeader, \
@@ -96,14 +96,6 @@ def get_all_packets_by_reg(packets):
     return packet_list
 
 
-def get_body(packet):
-    try:
-        return "".join([item for item in packet.split("\r\n\r\n") if "HTTP/" not in item])
-    except Exception:
-        traceback.print_exc()
-        return ""
-
-
 def get_header_value(header_set, value):
     result = [item for item in header_set if value in item]
     if len(result) != 0:
@@ -112,23 +104,20 @@ def get_header_value(header_set, value):
         return ""
 
 
-def get_detail_by_package(packets_from_pcap, publicField, use_regx):
+def get_detail_by_package(publicField, req_header, req_body, res_header, res_body):
     """
     通过pcap的数量分离session并完善相关字段
-    :param packets_from_pcap: 通过PcAp解析出的包
     :param publicField: 原始的session单条数据
+    :param req_header:请求头
+    :param req_body:请求体
+    :param res_header:响应头
+    :param res_body:响应体
     :return: 完整的单条数据
     """
-    res_field = publicField.copy()
-    if use_regx:
-        req = packets_from_pcap['req_body']
-        res = packets_from_pcap['res_body']
-    else:
-        res = packets_from_pcap["response"]
-        req = packets_from_pcap["request"]
+    res_field = copy.deepcopy(publicField)
     res_field["initRTT"] = firstOrZero(res_field.get("initRTT", 0))
     res_field["length"] = firstOrZero(res_field.get("length", 0))
-    request_lines = req.strip().split("\n")
+    request_lines = req_header.strip().split("\n")
     http_request_lines = [item for item in request_lines if "HTTP" in item]
     if len(http_request_lines) != 0:
         first_line = http_request_lines[0].split(" ")
@@ -144,15 +133,13 @@ def get_detail_by_package(packets_from_pcap, publicField, use_regx):
                                                               value="Content-Type")
     res_field['http.hostTokens'] = get_header_value(header_set=request_lines, value="Host")
 
-    if use_regx:
-        res_field['plain_body_src'] = ""
-        res_field['plain_body_dst'] = ""
-        if content_type_is_plain(req):
-            res_field['plain_body_src'] = get_body(req)
-        if content_type_is_plain(res):
-            res_field['plain_body_dst'] = get_body(res)
-
-    response_lines = res.strip().split("\n")
+    res_field['plain_body_src'] = ""
+    res_field['plain_body_dst'] = ""
+    if content_type_is_plain(req_header):
+        res_field['plain_body_src'] = req_body
+    if content_type_is_plain(res_header):
+        res_field['plain_body_dst'] = res_body
+    response_lines = res_body.strip().split("\n")
     http_response_lines = [item for item in response_lines if "HTTP" in item]
     if len(http_response_lines) != 0:
         first_line = http_response_lines[0].strip().split(" ")

xbase_util-0.6.7/xbase_util/segment.py

@@ -0,0 +1,163 @@
+import copy
+import re
+
+import numpy as np
+from scapy.all import *
+from scapy.layers.inet import TCP
+
+REQUEST_LINE_RE = re.compile(rb"^(GET|POST|PUT|DELETE|OPTIONS|HEAD|PATCH)\s[^\r\n]+\r\n", re.MULTILINE)
+RESPONSE_LINE_RE = re.compile(rb"^HTTP/\d\.\d\s+\d{3}\s?[^\r\n]*", re.IGNORECASE)
+
+
+def read_packets(packets):
+    last_seq_len = -1
+    last_ack = -1
+    packet_list = []
+    tmp_data = b''
+    tmp_packets = []
+    for index, pkt in enumerate(packets):
+        data = pkt[Raw].load if Raw in pkt else b''
+        ack = pkt[TCP].ack
+        seq = pkt[TCP].seq
+        if seq == last_seq_len:
+            # print(f"检测到连续包 数据长度:{len(data)} + seq:{seq}={len(data) + seq}  ack:{ack}")
+            tmp_data += data
+            tmp_packets.append(pkt)
+        elif seq == last_ack:
+            if tmp_data != b'':
+                if REQUEST_LINE_RE.match(tmp_data) or RESPONSE_LINE_RE.match(tmp_data):
+                    packet_list.append({'data': copy.deepcopy(tmp_data), 'pkts': copy.deepcopy(tmp_packets)})
+                else:
+                    # print("没有新的请求或者响应，就把数据加到上一个里面")
+                    if len(packet_list) > 0:
+                        # 之前找到过有请求，可以添加到之前的数据，否则说明一开始就没找到请求
+                        packet_list[-1]['pkts'].extend(copy.deepcopy(tmp_packets))
+                        packet_list[-1]['data'] += tmp_data
+
+            tmp_data = data
+            tmp_packets = [pkt]
+            # print(f"顺序正确 数据长度:{len(data)} + seq:{seq}={len(data) + seq}  ack:{ack}")
+        else:
+            # print(f"顺序错误 数据长度:{len(data)} + seq:{seq}={len(data) + seq}  ack:{ack}")
+            if len(data) > 0:
+                # 但是有数据
+                tmp_data += data
+                tmp_packets.append(pkt)
+        last_ack = ack
+        last_seq_len = seq + len(data)
+    if tmp_data != b'':
+        packet_list.append({'data': copy.deepcopy(tmp_data), 'pkts': copy.deepcopy(tmp_packets)})
+        tmp_packets.clear()
+    return packet_list
+
+
+def parse_req_or_res(data, pkts):
+    if data.find(b"\r\n\r\n") != -1:
+        res = data.split(b"\r\n\r\n", 1)
+        header = res[0]
+        body = res[1]
+    else:
+        header = data
+        body = b''
+    pattern_chuncked = re.compile(rb"Transfer-Encoding:\s*chunked", re.IGNORECASE)
+    pattern_gzip = re.compile(rb"Content-Encoding:\s*gzip", re.IGNORECASE)
+    chuncked_pattern = pattern_chuncked.search(header)
+    gzip_pattern = pattern_gzip.search(header)
+    if chuncked_pattern and b'chunked' in chuncked_pattern.group():
+        chunk_lines = [item for item in body.split(b"\r\n") if item != b'']
+        data = b''
+        next_chunk_size = 0
+        for chunk in chunk_lines:
+            try:
+                next_chunk_size = int(chunk, 16)
+                if next_chunk_size == 0:
+                    break
+                # print(f"接下来的分段大小：{next_chunk_size}")
+            except:
+                if next_chunk_size > 0:
+                    data += chunk
+                    # print(f"分段数据大小：{len(data)}")
+        result_body = data
+    else:
+        # print("虽然没有指定chunked，但是我猜出来他就是chunked")
+        if body.endswith(b"0\r\n"):
+            chunk_lines = [item for item in body.split(b"\r\n") if item != b'']
+            data = b''
+            next_chunk_size = 0
+            for chunk in chunk_lines:
+                try:
+                    next_chunk_size = int(chunk, 16)
+                    if next_chunk_size == 0:
+                        break
+                    # print(f"接下来的分段大小：{next_chunk_size}")
+                except:
+                    if next_chunk_size > 0:
+                        data += chunk
+                        # print(f"分段数据大小：{len(data)}")
+            result_body = data
+        else:
+            result_body = body
+    if gzip_pattern and b'gzip' in gzip_pattern.group():
+        try:
+            decompressed = gzip.decompress(result_body)
+            result_body_str = "\n".join(
+                [line.strip() for line in decompressed.decode("utf-8", errors="replace").splitlines() if
+                 line.strip() != ""])
+        except Exception as e:
+            result_body_str = result_body.decode("utf-8", errors="replace")
+    else:
+        result_body_str = result_body.decode("utf-8", errors="replace")
+    return header.decode("utf-8", errors="replace"), result_body_str, [float(pkt.time) for pkt in pkts]
+
+
+def get_all_packets_by_segment(packets):
+    res = read_packets(packets)
+    request_packets = [item for item in res if REQUEST_LINE_RE.match(item['data'])]
+    response_packets = [
+        {'first_seq': item['pkts'][0][TCP].seq, 'pkts': item['pkts'], 'first_ack': item['pkts'][0][TCP].ack,
+         'data': item['data']} for item in
+        res if RESPONSE_LINE_RE.match(item['data'])]
+    packet_list = []
+    for request in request_packets:
+        pkt_list = request['pkts']
+        last_pkt = pkt_list[-1]
+        # seq = last_pkt[TCP].seq
+        ack = last_pkt[TCP].ack
+        response = [item for item in response_packets if item['first_seq'] == ack]
+        # print(f"找到对应的响应：{len(response)}")
+        # print(f"请求：{request['data'].decode('utf-8', errors='replace')}")
+        if len(response) > 0:
+            res_header, res_body, res_times = parse_req_or_res(response[0]['data'], response[0]['pkts'])
+            req_header, req_body, req_times = parse_req_or_res(request['data'], request['pkts'])
+            packet_list.append({
+                "req_header": req_header,
+                "req_body": req_body,
+                "req_time": req_times,
+                "req_packets": len(request['pkts']),
+                "res_header": res_header,
+                "res_body": res_body,
+                "res_time": res_times,
+                "res_packets": len(response[0]['pkts']),
+            })
+        else:
+            # print("没响应")
+            req_header, req_body, req_times = parse_req_or_res(request['data'], request['pkts'])
+            packet_list.append({
+                "req_header": req_header,
+                "req_body": req_body,
+                "req_time": req_times,
+                "req_packets": len(request['pkts']),
+                "res_header": '',
+                "res_body": '',
+                "res_time": [],
+                "res_packets": 0,
+            })
+    return packet_list
+
+
+# if __name__ == '__main__':
+    # all_packets = get_all_packets_by_segment(rdpcap("../out/3post.pcap"))
+    # res=[
+    #     get_detail_by_package({}, package['req_header'], package['req_body'], package['res_header'],
+    #                           package['req_body']) for package in all_packets]
+    # print(res)

{xbase_util-0.6.5 → xbase_util-0.6.7}/xbase_util.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase-util
-Version: 0.6.5
+Version: 0.6.7
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt

{xbase_util-0.6.5 → xbase_util-0.6.7}/xbase_util.egg-info/SOURCES.txt

@@ -9,6 +9,7 @@ xbase_util/geo_util.py
 xbase_util/handle_features_util.py
 xbase_util/packet_util.py
 xbase_util/pcap_util.py
+xbase_util/segment.py
 xbase_util/xbase_constant.py
 xbase_util/xbase_util.py
 xbase_util.egg-info/PKG-INFO