xbase-util 0.7.3__tar.gz → 0.7.5__tar.gz

{xbase_util-0.7.3 → xbase_util-0.7.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase_util
-Version: 0.7.3
+Version: 0.7.5
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt

{xbase_util-0.7.3 → xbase_util-0.7.5}/setup.py

@@ -3,7 +3,7 @@ from distutils.core import setup
 from setuptools import find_packages
 
 setup(name="xbase_util",
-      version="0.7.3",
+      version="0.7.5",
       description="网络安全基础工具",
       long_description="包含提取，预测，训练的基础工具",
       author="xyt",

xbase_util-0.7.5/test/test.py

@@ -0,0 +1,34 @@
+import copy
+import gzip
+import pickle
+import re
+import traceback
+
+from xbase_util.packet_util import filter_visible_chars
+from xbase_util.pcap_util import reassemble_tcp, reassemble_session
+
+if __name__ == '__main__':
+    # req = EsReq("http://127.0.0.1:9200")
+    # exp=build_es_expression(size="1",
+    #                     start_time=None,
+    #                     end_time=None,
+    #                     arkime_expression='id == 250106-lKoC7T_SwbNAe4xDQQx7KTOd')
+    # session=req.search(body=exp,index="arkime_sessions3-*").json()['hits']['hits'][0]
+    # packetPos=session['_source']['packetPos']
+    # stream,packet_objs=process_session_id_disk_simple(id=session['_id'], node=session['_source']['node'],
+    #                                packet_pos=packetPos, esdb=EsDb(req, multiprocessing.Manager()),
+    #                                pcap_path_prefix="origin")
+    #
+    # with open('stream.pkl', 'wb') as f:
+    #     pickle.dump(stream, f)
+    # with open('packet_objs.pkl', 'wb') as f:
+    #     pickle.dump(packet_objs, f)
+
+    with open('stream.pkl', 'rb') as f:
+        stream = pickle.load(f)
+    with open('packet_objs.pkl', 'rb') as f:
+        packet_objs = pickle.load(f)
+    skey = f"10.28.7.16:54398"
+    reassemble_tcp_res = reassemble_tcp(packet_objs, skey)
+    reassemble_session_result = reassemble_session(reassemble_tcp_res, skey)
+    print(reassemble_session_result)

{xbase_util-0.7.3 → xbase_util-0.7.5}/xbase_util/packet_util.py

@@ -1,11 +1,8 @@
 import copy
 import re
 
-from scapy.layers.inet import TCP
-from scapy.packet import Raw
-
 from xbase_util.xbase_constant import plain_content_type_columns, packetKeyname, src_dst_header, statisticHeader, \
-    features_key, plain_body_columns
+    features_key, plain_body_columns, http_version_pattern, http_req_method_pattern, http_req_path_pattern, res_status_code_pattern
 from xbase_util.xbase_util import firstOrZero
 
 
@@ -59,47 +56,11 @@ def get_all_columns(
     return result_columns
 
 
-req_pattern = re.compile(r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?\r\n\r\n",
-                         re.DOTALL)
-res_pattern = re.compile(r"HTTP/\d\.\d \d{3}.*", re.DOTALL)
-req_body_pattern = re.compile(
-    r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?(?=HTTP/\d\.\d)", re.DOTALL)
-
-http_version = re.compile(r"HTTP\/(\d\.\d)")
-http_req_method = re.compile(r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d")
-http_req_path = re.compile(r"(?:GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH)\s+(\/[^\s]*)\s+HTTP\/\d\.\d")
-res_status_code_pattern = re.compile(r"HTTP\/\d\.\d\s+(\d{3})\s+.*")
-
-
-# def get_all_packets_by_reg(packets):
-#     http_Req_Raw = {}
-#     http_methods = ("POST /", "PUT /", "OPTIONS /", "DELETE /", "GET /")
-#     for packet in packets:
-#         if TCP in packet and Raw in packet:
-#             data = packet[Raw].load
-#             ack = packet[TCP].ack
-#             next_ack = packet[TCP].seq + len(data)
-#             if ack not in http_Req_Raw:
-#                 http_Req_Raw[ack] = {"time": [float(packet.time)], "data": data, "next_ack": next_ack}
-#             else:
-#                 http_Req_Raw[ack]["time"].append(float(packet.time))
-#                 http_Req_Raw[ack]["data"] += data
-#                 http_Req_Raw[ack]["next_ack"] = next_ack
-#     packet_list = [
-#         {
-#             'req_data': item['data'],
-#             'res_data': http_Req_Raw[item['next_ack']]['data'],
-#             'req_text': filter_visible_chars(item['data']),
-#             'res_text': filter_visible_chars(http_Req_Raw[item['next_ack']]['data']),
-#             'req_time': item['time'],
-#             'res_time': http_Req_Raw[item['next_ack']]['time']
-#         }
-#         for ack, item in http_Req_Raw.items()
-#         if item['next_ack'] in http_Req_Raw  # 检查 next_ack 是否存在
-#            and any(method in filter_visible_chars(item['data']) for method in http_methods)
-#     ]
-#     return packet_list
-
+# req_pattern = re.compile(r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?\r\n\r\n",
+#                          re.DOTALL)
+# res_pattern = re.compile(r"HTTP/\d\.\d \d{3}.*", re.DOTALL)
+# req_body_pattern = re.compile(
+#     r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?(?=HTTP/\d\.\d)", re.DOTALL)
 
 def get_header_value(header_set, value):
     result = [item for item in header_set if value in item]
@@ -123,10 +84,10 @@ def get_detail_by_package(publicField, req_header, req_body, res_header, res_bod
     res_field["initRTT"] = firstOrZero(res_field.get("initRTT", 0))
     res_field["length"] = firstOrZero(res_field.get("length", 0))
 
-    http_version_res = http_version.findall(req_header)
+    http_version_res = http_version_pattern.findall(req_header)
     res_field['http.clientVersion'] = http_version_res[0] if len(http_version_res) > 0 else ""
-    http_method = http_req_method.findall(req_header)
-    http_path = http_req_path.findall(req_header)
+    http_method = http_req_method_pattern.findall(req_header)
+    http_path = http_req_path_pattern.findall(req_header)
     res_field['http.clientVersion'] = http_version_res[0] if len(http_version_res) > 0 else ""
     res_field['http.method'] = http_method[0] if len(http_method) > 0 else ""
     res_field['http.path'] = http_path[0] if len(http_path) > 0 else ""
@@ -139,11 +100,11 @@ def get_detail_by_package(publicField, req_header, req_body, res_header, res_bod
     res_field['plain_body_src'] = ""
     res_field['plain_body_dst'] = ""
     if content_type_is_plain(req_header):
-        res_field['plain_body_src'] = filter_visible_chars(f'{req_body}')
+        res_field['plain_body_src'] = req_body
     if content_type_is_plain(res_header):
-        res_field['plain_body_dst'] = filter_visible_chars(f'{res_body}')
+        res_field['plain_body_dst'] = res_body
 
-    http_server_version_res = http_version.findall(res_header)
+    http_server_version_res = http_version_pattern.findall(res_header)
     res_field['http.serverVersion'] = http_server_version_res[0] if len(http_server_version_res) > 0 else ""
 
     status_code = res_status_code_pattern.findall(res_header)