xbase-util 0.7.4__tar.gz → 0.7.6__tar.gz

{xbase_util-0.7.4 → xbase_util-0.7.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase_util
-Version: 0.7.4
+Version: 0.7.6
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt

{xbase_util-0.7.4 → xbase_util-0.7.6}/setup.py

@@ -3,7 +3,7 @@ from distutils.core import setup
 from setuptools import find_packages
 
 setup(name="xbase_util",
-      version="0.7.4",
+      version="0.7.6",
       description="网络安全基础工具",
       long_description="包含提取，预测，训练的基础工具",
       author="xyt",

xbase_util-0.7.6/test/test.py

@@ -0,0 +1,36 @@
+import copy
+import gzip
+import pickle
+import re
+import traceback
+
+from xbase_util.packet_util import filter_visible_chars
+from xbase_util.pcap_util import reassemble_tcp, reassemble_session
+
+if __name__ == '__main__':
+    # req = EsReq("http://127.0.0.1:9200")
+    # exp=build_es_expression(size="1",
+    #                     start_time=None,
+    #                     end_time=None,
+    #                     arkime_expression='id == 250106-lKoC7T_SwbNAe4xDQQx7KTOd')
+    # session=req.search(body=exp,index="arkime_sessions3-*").json()['hits']['hits'][0]
+    # packetPos=session['_source']['packetPos']
+    # stream,packet_objs=process_session_id_disk_simple(id=session['_id'], node=session['_source']['node'],
+    #                                packet_pos=packetPos, esdb=EsDb(req, multiprocessing.Manager()),
+    #                                pcap_path_prefix="origin")
+    #
+    # with open('stream.pkl', 'wb') as f:
+    #     pickle.dump(stream, f)
+    # with open('packet_objs.pkl', 'wb') as f:
+    #     pickle.dump(packet_objs, f)
+
+    with open('stream.pkl', 'rb') as f:
+        stream = pickle.load(f)
+    with open('packet_objs.pkl', 'rb') as f:
+        packet_objs = pickle.load(f)
+    skey = f"10.28.7.16:54398"
+    reassemble_tcp_res = reassemble_tcp(packet_objs, skey)
+    all_packets = reassemble_session(reassemble_tcp_res, skey)
+    time_period = [( abs(item['res_time']-item['req_time'])) for item in
+                   all_packets if item['res_time'] != 0 and item['req_time'] != 0]
+    print(all_packets)

{xbase_util-0.7.4 → xbase_util-0.7.6}/xbase_util/packet_util.py

@@ -1,11 +1,8 @@
 import copy
 import re
 
-from scapy.layers.inet import TCP
-from scapy.packet import Raw
-
 from xbase_util.xbase_constant import plain_content_type_columns, packetKeyname, src_dst_header, statisticHeader, \
-    features_key, plain_body_columns
+    features_key, plain_body_columns, http_version_pattern, http_req_method_pattern, http_req_path_pattern, res_status_code_pattern
 from xbase_util.xbase_util import firstOrZero
 
 
@@ -59,16 +56,11 @@ def get_all_columns(
     return result_columns
 
 
-req_pattern = re.compile(r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?\r\n\r\n",
-                         re.DOTALL)
-res_pattern = re.compile(r"HTTP/\d\.\d \d{3}.*", re.DOTALL)
-req_body_pattern = re.compile(
-    r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?(?=HTTP/\d\.\d)", re.DOTALL)
-
-http_version = re.compile(r"HTTP\/(\d\.\d)")
-http_req_method = re.compile(r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d")
-http_req_path = re.compile(r"(?:GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH)\s+(\/[^\s]*)\s+HTTP\/\d\.\d")
-res_status_code_pattern = re.compile(r"HTTP\/\d\.\d\s+(\d{3})\s+.*")
+# req_pattern = re.compile(r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?\r\n\r\n",
+#                          re.DOTALL)
+# res_pattern = re.compile(r"HTTP/\d\.\d \d{3}.*", re.DOTALL)
+# req_body_pattern = re.compile(
+#     r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?(?=HTTP/\d\.\d)", re.DOTALL)
 
 def get_header_value(header_set, value):
     result = [item for item in header_set if value in item]
@@ -92,10 +84,10 @@ def get_detail_by_package(publicField, req_header, req_body, res_header, res_bod
     res_field["initRTT"] = firstOrZero(res_field.get("initRTT", 0))
     res_field["length"] = firstOrZero(res_field.get("length", 0))
 
-    http_version_res = http_version.findall(req_header)
+    http_version_res = http_version_pattern.findall(req_header)
     res_field['http.clientVersion'] = http_version_res[0] if len(http_version_res) > 0 else ""
-    http_method = http_req_method.findall(req_header)
-    http_path = http_req_path.findall(req_header)
+    http_method = http_req_method_pattern.findall(req_header)
+    http_path = http_req_path_pattern.findall(req_header)
     res_field['http.clientVersion'] = http_version_res[0] if len(http_version_res) > 0 else ""
     res_field['http.method'] = http_method[0] if len(http_method) > 0 else ""
     res_field['http.path'] = http_path[0] if len(http_path) > 0 else ""
@@ -112,7 +104,7 @@ def get_detail_by_package(publicField, req_header, req_body, res_header, res_bod
     if content_type_is_plain(res_header):
         res_field['plain_body_dst'] = res_body
 
-    http_server_version_res = http_version.findall(res_header)
+    http_server_version_res = http_version_pattern.findall(res_header)
     res_field['http.serverVersion'] = http_server_version_res[0] if len(http_server_version_res) > 0 else ""
 
     status_code = res_status_code_pattern.findall(res_header)