xbase-util 0.8.0__tar.gz → 0.8.2__tar.gz

{xbase_util-0.8.0 → xbase_util-0.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase_util
-Version: 0.8.0
+Version: 0.8.2
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt

{xbase_util-0.8.0 → xbase_util-0.8.2}/setup.py

@@ -3,7 +3,7 @@ from distutils.core import setup
 from setuptools import find_packages
 
 setup(name="xbase_util",
-      version="0.8.0",
+      version="0.8.2",
       description="网络安全基础工具",
       long_description="包含提取，预测，训练的基础工具",
       author="xyt",

{xbase_util-0.8.0 → xbase_util-0.8.2}/xbase_util/pcap_util.py

@@ -56,6 +56,8 @@ def decompress_streaming(compressed_data, session_id):
         return bytearray()
 
 
+
+
 def reassemble_tcp(packets, num_packets=1000):
     packets2 = []
     info = {}
@@ -121,7 +123,7 @@ def reassemble_tcp(packets, num_packets=1000):
         return a['tcp']['ack'] - (b['tcp']['seq'] + len(b['tcp']['data']) - 1)
 
     packets.sort(key=cmp_to_key(compare_packets))
-    del packets[num_packets:]
+    # del packets[num_packets:]
     # Now divide up conversation
     clientSeq = 0
     hostSeq = 0
@@ -771,7 +773,7 @@ def decode_obj(buffer, bigEndian, linkType, nanosecond):
     return obj
 
 
-def get_file_and_read_pos(session_id, file, pos_list):
+def get_file_and_read_pos(session_id, file, pos_list,reture_obj=True):
     filename = file['name']
     if not os.path.isfile(filename):
         print(f"文件不存在:{filename}")
@@ -822,8 +824,9 @@ def get_file_and_read_pos(session_id, file, pos_list):
     i = 0
     for pos in pos_list:
         packet_bytes = read_packet(pos, param_map, session_id)
-        obj = decode_obj(packet_bytes, bigEndian, linkType, nanosecond, )
-        packet_objs.append(copy.deepcopy(obj))
+        if reture_obj:
+            obj = decode_obj(packet_bytes, bigEndian, linkType, nanosecond, )
+            packet_objs.append(copy.deepcopy(obj))
         if not packet_bytes:
             continue
         packets[i] = packet_bytes
@@ -845,7 +848,7 @@ def get_file_and_read_pos(session_id, file, pos_list):
     return res, packet_objs
 
 
-def process_session_id_disk_simple(id, node, packet_pos, esdb, pcap_path_prefix):
+def process_session_id_disk_simple(id, node, packet_pos, esdb, pcap_path_prefix,reture_obj=True):
     packetPos = packet_pos
     file = esdb.get_file_by_file_id(node=node, num=abs(packetPos[0]),
                                     prefix=None if pcap_path_prefix == "origin" else pcap_path_prefix)
@@ -854,4 +857,4 @@ def process_session_id_disk_simple(id, node, packet_pos, esdb, pcap_path_prefix)
     fix_pos(packetPos, file['packetPosEncoding'])
     pos_list = group_numbers(packetPos)[0]
     pos_list.pop(0)
-    return get_file_and_read_pos(id, file, pos_list)
+    return get_file_and_read_pos(id, file, pos_list,reture_obj)

xbase_util-0.8.2/xbase_util/segment.py

@@ -0,0 +1,105 @@
+from scapy.all import *
+from scapy.layers.inet import TCP
+
+from xbase_util.packet_util import filter_visible_chars
+from xbase_util.xbase_util import parse_chunked_body
+
+REQUEST_LINE_RE = re.compile(rb"^(GET|POST|PUT|DELETE|OPTIONS|HEAD|PATCH)\s[^\r\n]+\r\n", re.MULTILINE)
+RESPONSE_LINE_RE = re.compile(rb"^HTTP/\d\.\d\s+\d{3}\s?[^\r\n]*", re.IGNORECASE)
+
+
+def read_packets(packets):
+    last_seq_len = -1
+    last_ack = -1
+    packet_list = []
+    tmp_data = b''
+    tmp_packets = []
+    for index, pkt in enumerate(packets):
+        data = pkt[Raw].load if Raw in pkt else b''
+        ack = pkt[TCP].ack
+        seq = pkt[TCP].seq
+        if seq == last_seq_len:
+            # print(f"检测到连续包 数据长度:{len(data)} + seq:{seq}={len(data) + seq}  ack:{ack}")
+            tmp_data += data
+            tmp_packets.append(pkt)
+        elif seq == last_ack:
+            if tmp_data != b'':
+                if REQUEST_LINE_RE.match(tmp_data) or RESPONSE_LINE_RE.match(tmp_data):
+                    packet_list.append({'data': copy.deepcopy(tmp_data), 'pkts': copy.deepcopy(tmp_packets)})
+                else:
+                    # print("没有新的请求或者响应，就把数据加到上一个里面")
+                    if len(packet_list) > 0:
+                        # 之前找到过有请求，可以添加到之前的数据，否则说明一开始就没找到请求
+                        packet_list[-1]['pkts'].extend(copy.deepcopy(tmp_packets))
+                        packet_list[-1]['data'] += tmp_data
+
+            tmp_data = data
+            tmp_packets = [pkt]
+            # print(f"顺序正确 数据长度:{len(data)} + seq:{seq}={len(data) + seq}  ack:{ack}")
+        else:
+            # print(f"顺序错误 数据长度:{len(data)} + seq:{seq}={len(data) + seq}  ack:{ack}")
+            if len(data) > 0:
+                # 但是有数据
+                tmp_data += data
+                tmp_packets.append(pkt)
+        last_ack = ack
+        last_seq_len = seq + len(data)
+    if tmp_data != b'':
+        packet_list.append({'data': copy.deepcopy(tmp_data), 'pkts': copy.deepcopy(tmp_packets)})
+        tmp_packets.clear()
+    return packet_list
+
+
+def parse_req_or_res(data, pkts):
+    if data.find(b"\r\n\r\n") != -1:
+        res = data.split(b"\r\n\r\n", 1)
+        header = res[0]
+        body = res[1]
+    else:
+        header = data
+        body = b''
+    body = parse_chunked_body(body)
+    result_body_str = filter_visible_chars(body)
+    return filter_visible_chars(header), result_body_str, [float(pkt.time) for pkt in pkts]
+
+
+def get_all_packets_by_segment(packets):
+    res = read_packets(packets)
+    request_packets = [item for item in res if REQUEST_LINE_RE.match(item['data'])]
+    response_packets = [
+        {'first_seq': item['pkts'][0][TCP].seq, 'pkts': item['pkts'], 'first_ack': item['pkts'][0][TCP].ack,
+         'data': item['data']} for item in
+        res if RESPONSE_LINE_RE.match(item['data'])]
+    packet_list = []
+    for request in request_packets:
+        pkt_list = request['pkts']
+        last_pkt = pkt_list[-1]
+        ack = last_pkt[TCP].ack
+        response = [item for item in response_packets if item['first_seq'] == ack]
+        if len(response) > 0:
+            res_header, res_body, res_times = parse_req_or_res(response[0]['data'], response[0]['pkts'])
+            req_header, req_body, req_times = parse_req_or_res(request['data'], request['pkts'])
+            packet_list.append({
+                "req_header": req_header,
+                "req_body": req_body,
+                "req_time": req_times,
+                "req_packets": len(request['pkts']),
+                "res_header": res_header,
+                "res_body": res_body,
+                "res_time": res_times,
+                "res_packets": len(response[0]['pkts']),
+            })
+        else:
+            # print("没响应")
+            req_header, req_body, req_times = parse_req_or_res(request['data'], request['pkts'])
+            packet_list.append({
+                "req_header": req_header,
+                "req_body": req_body,
+                "req_time": req_times,
+                "req_packets": len(request['pkts']),
+                "res_header": '',
+                "res_body": '',
+                "res_time": [],
+                "res_packets": 0,
+            })
+    return packet_list

{xbase_util-0.8.0 → xbase_util-0.8.2}/xbase_util/xbase_util.py

@@ -1,3 +1,4 @@
+import gzip
 import json
 import logging
 import os
@@ -11,7 +12,28 @@ import numpy as np
 import tldextract
 from scapy.layers.dns import DNS
 
-from xbase_util.xbase_constant import parse_path, dns_domain_list
+from xbase_util.xbase_constant import dns_domain_list, parse_path
+
+
+def parse_chunked_body(data: bytes) -> bytes:
+    body = b""
+    while True:
+        chunk_size_end = data.find(b"\r\n")
+        if chunk_size_end == -1:
+            break
+        chunk_size_hex = data[:chunk_size_end]
+        chunk_size = int(chunk_size_hex, 16)
+        if chunk_size == 0:
+            break
+        chunk_start = chunk_size_end + 2
+        chunk_end = chunk_start + chunk_size
+        body += data[chunk_start:chunk_end]
+        data = data[chunk_end + 2:]
+    try:
+        body = gzip.decompress(body)
+        return body
+    except gzip.BadGzipFile:
+        return body
 
 
 def process_origin_pos(originPos):

{xbase_util-0.8.0 → xbase_util-0.8.2}/xbase_util.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase-util
-Version: 0.8.0
+Version: 0.8.2
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt

{xbase_util-0.8.0 → xbase_util-0.8.2}/xbase_util.egg-info/SOURCES.txt

@@ -1,6 +1,5 @@
 README.md
 setup.py
-test/test.py
 xbase_util/__init__.py
 xbase_util/add_column_util.py
 xbase_util/dangerous_util.py
@@ -10,6 +9,7 @@ xbase_util/geo_util.py
 xbase_util/handle_features_util.py
 xbase_util/packet_util.py
 xbase_util/pcap_util.py
+xbase_util/segment.py
 xbase_util/xbase_constant.py
 xbase_util/xbase_util.py
 xbase_util.egg-info/PKG-INFO

xbase_util-0.8.0/test/test.py

@@ -1,38 +0,0 @@
-import copy
-import gzip
-import pickle
-import re
-import traceback
-
-from requests import session
-
-from xbase_util.packet_util import filter_visible_chars
-from xbase_util.pcap_util import reassemble_tcp, reassemble_session
-
-if __name__ == '__main__':
-    # req = EsReq("http://127.0.0.1:9200")
-    # exp=build_es_expression(size="1",
-    #                     start_time=None,
-    #                     end_time=None,
-    #                     arkime_expression='id == 250106-lKoC7T_SwbNAe4xDQQx7KTOd')
-    # session=req.search(body=exp,index="arkime_sessions3-*").json()['hits']['hits'][0]
-    # packetPos=session['_source']['packetPos']
-    # stream,packet_objs=process_session_id_disk_simple(id=session['_id'], node=session['_source']['node'],
-    #                                packet_pos=packetPos, esdb=EsDb(req, multiprocessing.Manager()),
-    #                                pcap_path_prefix="origin")
-    #
-    # with open('stream.pkl', 'wb') as f:
-    #     pickle.dump(stream, f)
-    # with open('packet_objs.pkl', 'wb') as f:
-    #     pickle.dump(packet_objs, f)
-
-    with open('stream.pkl', 'rb') as f:
-        stream = pickle.load(f)
-    with open('packet_objs.pkl', 'rb') as f:
-        packet_objs = pickle.load(f)
-    skey = f"10.28.7.16:54398"
-    reassemble_tcp_res = reassemble_tcp(packet_objs, skey)
-    all_packets = reassemble_session(reassemble_tcp_res, skey,session_id="emm")
-    time_period = [( abs(item['res_time']-item['req_time'])) for item in
-                   all_packets if item['res_time'] != 0 and item['req_time'] != 0]
-    print(all_packets)