xbase-util 0.8.5__tar.gz → 0.8.7__tar.gz

{xbase_util-0.8.5 → xbase_util-0.8.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase_util
-Version: 0.8.5
+Version: 0.8.7
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt

{xbase_util-0.8.5 → xbase_util-0.8.7}/setup.py

@@ -3,7 +3,7 @@ from distutils.core import setup
 from setuptools import find_packages
 
 setup(name="xbase_util",
-      version="0.8.5",
+      version="0.8.7",
       description="网络安全基础工具",
       long_description="包含提取，预测，训练的基础工具",
       author="xyt",

{xbase_util-0.8.5 → xbase_util-0.8.7}/xbase_util/common_util.py

@@ -25,28 +25,33 @@ def filter_visible_chars(data):
     return ''.join(chr(b) for b in data if 32 <= b <= 126 or b in (9, 10, 13))
 
 
-def parse_chunked_body(data: bytes, need_un_gzip=False) -> bytes:
-    body = b''
-    while True:
-        chunk_size_end = data.find(b"\r\n")
-        if chunk_size_end == -1:
-            break
-        chunk_size_hex = data[:chunk_size_end]
-        chunk_size = int(chunk_size_hex, 16)
-        if chunk_size == 0:
-            break
-        chunk_start = chunk_size_end + 2
-        chunk_end = chunk_start + chunk_size
-        body += data[chunk_start:chunk_end]
-        data = data[chunk_end + 2:]
-    if need_un_gzip:
-        try:
-            return gzip.decompress(body)
-        except gzip.BadGzipFile:
-            print("解压错误")
+def parse_chunked_body(data: bytes, need_un_gzip=False,session_id="none") -> bytes:
+    try:
+        body = b''
+        while True:
+            chunk_size_end = data.find(b"\r\n")
+            if chunk_size_end == -1:
+                break
+            chunk_size_hex = data[:chunk_size_end]
+            print(f"chunk_size_hex:{chunk_size_hex}")
+            chunk_size = int(chunk_size_hex, 16)
+            if chunk_size == 0:
+                break
+            chunk_start = chunk_size_end + 2
+            chunk_end = chunk_start + chunk_size
+            body += data[chunk_start:chunk_end]
+            data = data[chunk_end + 2:]
+        if need_un_gzip:
+            try:
+                return gzip.decompress(body)
+            except gzip.BadGzipFile:
+                print("解压错误")
+                return body
+        else:
             return body
-    else:
-        return body
+    except Exception as e:
+        print(f"parse_chunked_body失败，session:{session_id}")
+        return b''
 
 
 def process_origin_pos(originPos):
@@ -86,7 +91,6 @@ def get_ua_duplicate_count(all_packets):
         ua_list.extend(lines)
     counter = Counter(ua_list)
     pairs = sum(count // 2 for count in counter.values())
-    print(pairs)
     return pairs
 
 

{xbase_util-0.8.5 → xbase_util-0.8.7}/xbase_util/pcap_util.py

@@ -274,7 +274,7 @@ def process_session_id_disk_simple(id, node, packet_pos, esdb, pcap_path_prefix)
     return get_file_and_read_pos(id, file, pos_list)
 
 
-def parse_body(data):
+def parse_body(data,session_id='none'):
     if data.find(b"\r\n\r\n") != -1:
         res = data.split(b"\r\n\r\n", 1)
         header = res[0]
@@ -286,7 +286,7 @@ def parse_body(data):
     gzip_pattern = pattern_gzip.search(header)
     need_gzip = gzip_pattern and b'gzip' in gzip_pattern.group()
     if chunked_pattern and b'chunked' in chunked_pattern.group():
-        body = parse_chunked_body(body, need_un_gzip=need_gzip)
+        body = parse_chunked_body(body, need_un_gzip=need_gzip,session_id=session_id)
     elif need_gzip:
         try:
             body = gzip.decompress(body)
@@ -297,7 +297,7 @@ def parse_body(data):
     return filter_visible_chars(header), result_body_str
 
 
-def reassemble_session_pcap(reassemble_tcp_res, skey):
+def reassemble_session_pcap(reassemble_tcp_res, skey,session_id='none'):
     my_map = {
         'key': '',
         'req_header': '',
@@ -311,7 +311,7 @@ def reassemble_session_pcap(reassemble_tcp_res, skey):
     }
     packet_list = []
     for index, packet in enumerate(reassemble_tcp_res):
-        header, body = parse_body(packet['data'])
+        header, body = parse_body(packet['data'], session_id=session_id)
         if index == len(reassemble_tcp_res) - 1:
             packet_list.append(copy.deepcopy(my_map))
         if packet['key'] == skey:
@@ -366,8 +366,8 @@ def reassemble_tcp_pcap(p):
             info[key]['max'] = seq
         packets2.append(packet)
     if len(keys) == 1:
-        key = f"{packets['pkt'][IP].dst}:{packets['pkt'][IP].dport}"
-        ack = packets['pkt'][TCP].ack
+        key = f"{packets2[0]['pkt'][IP].dst}:{packets2[0]['pkt'][IP].dport}"
+        ack = packets2[0]['pkt'][TCP].ack
         info[key] = {
             "min": ack,
             "max": ack,
@@ -375,8 +375,7 @@ def reassemble_tcp_pcap(p):
             "wrapack": False,
         }
         keys.append(key)
-    packets = packets2
-    if len(packets) == 0:
+    if len(packets2) == 0:
         return []
     needwrap = False
     if info[keys[0]] and info[keys[0]]['max'] - info[keys[0]]['min'] > 0x7fffffff:
@@ -388,13 +387,13 @@ def reassemble_tcp_pcap(p):
         info[keys[0]]['wrapack'] = True
         needwrap = True
     if needwrap:
-        for packet in packets:
+        for packet in packets2:
             key = f"{packet['ip']['addr1']}:{packet['tcp']['sport']}"
             if info[key]['wrapseq'] and packet['tcp']['seq'] < 0x7fffffff:
                 packet['tcp']['seq'] += 0xffffffff
             if info[key]['wrapack'] and packet['tcp']['ack'] < 0x7fffffff:
                 packet['tcp']['ack'] += 0xffffffff
-    clientKey = f"{packets[0]['pkt'][IP].src}:{packets[0]['pkt'][IP].sport}"
+    clientKey = f"{packets2[0]['pkt'][IP].src}:{packets2[0]['pkt'][IP].sport}"
 
     def compare_packets(a, b):
         a_seq = a['pkt'][TCP].seq
@@ -413,14 +412,14 @@ def reassemble_tcp_pcap(p):
             return (a_seq + len(a_data) - 1) - b_ack
         return a_ack - (b_seq + len(b_data) - 1)
 
-    packets.sort(key=cmp_to_key(compare_packets))
+    packets2.sort(key=cmp_to_key(compare_packets))
     # del packets[num_packets:]
     # Now divide up conversation
     clientSeq = 0
     hostSeq = 0
     previous = 0
     results = []
-    for i, item in enumerate(packets):
+    for i, item in enumerate(packets2):
         sip = item['pkt'][IP].src
         sport = item['pkt'][IP].sport
         seq = item['pkt'][TCP].seq

{xbase_util-0.8.5 → xbase_util-0.8.7}/xbase_util/test.py

@@ -1,13 +1,7 @@
-import re
-from collections import Counter
-
-import numpy as np
 from scapy.packet import Raw
 from scapy.utils import rdpcap
 
-from xbase_util.common_util import get_res_status_code_list
 from xbase_util.pcap_util import reassemble_tcp_pcap, reassemble_session_pcap
-from xbase_util.xbase_constant import res_status_code_pattern
 
 if __name__ == '__main__':
     packets_scapy = reassemble_tcp_pcap(rdpcap("gzip2.pcap"))
@@ -17,5 +11,5 @@ if __name__ == '__main__':
         if Raw in pkt:
             streams += pkt[Raw].load
     text_data = streams.decode('ascii', errors='ignore')
-    all_packets = reassemble_session_pcap(packets_scapy, skey=skey)
+    all_packets = reassemble_session_pcap(packets_scapy, skey=skey,session_id='enn')
 

{xbase_util-0.8.5 → xbase_util-0.8.7}/xbase_util.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase-util
-Version: 0.8.5
+Version: 0.8.7
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt