xbase-util 0.4.6__tar.gz → 0.4.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {xbase_util-0.4.6 → xbase_util-0.4.7}/PKG-INFO +1 -1
- {xbase_util-0.4.6 → xbase_util-0.4.7}/setup.py +1 -1
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/packet_util.py +77 -34
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util.egg-info/PKG-INFO +1 -1
- {xbase_util-0.4.6 → xbase_util-0.4.7}/README.md +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/setup.cfg +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/__init__.py +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/add_column_util.py +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/dangerous_util.py +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/db/__init__.py +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/db/bean/ConfigBean.py +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/db/bean/CurrentConfigBean.py +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/db/bean/FlowBean.py +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/db/bean/TaskTemplateBean.py +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/db/bean/__init__.py +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/db/dao/ConfigDao.py +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/db/dao/CurrentConfigDao.py +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/db/dao/FlowDao.py +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/db/dao/TaskTemplateDao.py +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/db/dao/__init__.py +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/db/initsqlite3.py +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/es_db_util.py +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/esreq.py +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/geo_util.py +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/handle_features_util.py +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/pcap_util.py +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/xbase_constant.py +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/xbase_util.py +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util.egg-info/SOURCES.txt +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util.egg-info/dependency_links.txt +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util.egg-info/not-zip-safe +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util.egg-info/top_level.txt +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util_assets/GeoLite2-City.mmdb +0 -0
- {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util_assets/arkimeparse.js +0 -0
@@ -1,6 +1,7 @@
|
|
1
1
|
import re
|
2
2
|
|
3
3
|
from scapy.layers.inet import TCP
|
4
|
+
from scapy.packet import Raw
|
4
5
|
|
5
6
|
from xbase_util.xbase_constant import plain_content_type_columns, packetKeyname, src_dst_header, statisticHeader, \
|
6
7
|
features_key, plain_body_columns
|
@@ -57,41 +58,83 @@ def get_all_columns(
|
|
57
58
|
return result_columns
|
58
59
|
|
59
60
|
|
60
|
-
def
|
61
|
-
""
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
61
|
+
def get_all_packets_by_reg(packets):
|
62
|
+
req_pattern = re.compile(r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?\r\n\r\n",
|
63
|
+
re.DOTALL)
|
64
|
+
req_res_pattern = re.compile(
|
65
|
+
r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?(?=HTTP/\d\.\d \d{3} [a-zA-Z]+|$)",
|
66
|
+
re.DOTALL)
|
67
|
+
res_pattern = re.compile(r"HTTP/\d\.\d \d{3} [a-zA-Z]+.*", re.DOTALL)
|
68
|
+
tcp_packet_map = {}
|
69
|
+
for packet in packets:
|
70
|
+
if packet.haslayer(TCP) and packet.haslayer(Raw):
|
71
|
+
raw_data = bytes(packet[Raw].load)
|
72
|
+
ack = f"{packet[TCP].ack}"
|
73
|
+
seq = packet[TCP].seq
|
74
|
+
time = packet[TCP].time
|
75
|
+
if f"{packet[TCP].ack}" not in tcp_packet_map:
|
76
|
+
tcp_packet_map[ack] = {
|
77
|
+
"data": raw_data,
|
78
|
+
"time": [time],
|
79
|
+
"seq": [seq],
|
80
|
+
"last_seq": seq,
|
81
|
+
"ack": ack,
|
82
|
+
"len": [len(raw_data)],
|
83
|
+
"last_len": len(raw_data)
|
84
|
+
}
|
85
|
+
else:
|
86
|
+
tcp_packet_map[ack]['data'] += raw_data
|
87
|
+
tcp_packet_map[ack]['time'].append(time)
|
88
|
+
tcp_packet_map[ack]['seq'].append(seq)
|
89
|
+
tcp_packet_map[ack]['last_len'] = len(raw_data)
|
90
|
+
tcp_packet_map[ack]['last_seq'] = seq
|
91
|
+
packet_list = []
|
92
|
+
for ack, data_set in tcp_packet_map.items():
|
93
|
+
data_str = data_set['data'].decode("utf-8", errors="ignore")
|
94
|
+
request_re = re.search(req_pattern, data_str)
|
95
|
+
if request_re is None:
|
96
|
+
continue
|
97
|
+
next_ack = f"{data_set['last_len'] + data_set['last_seq']}"
|
98
|
+
packet_data = data_set['data']
|
99
|
+
request_time = data_set['time']
|
100
|
+
req_len = len(packet_data)
|
101
|
+
res_len = 0
|
102
|
+
while True:
|
103
|
+
# 持续往下一个包找,直到下一个包是请求为止,因为下一个包可能还是属于这个包的一部分,也可能是响应的一部分
|
104
|
+
# 下一个包的ack存在
|
105
|
+
if next_ack not in tcp_packet_map:
|
106
|
+
print("没找到新的ack")
|
107
|
+
break
|
108
|
+
new_packet = tcp_packet_map[next_ack]
|
109
|
+
# 判断新的包是不是响应包
|
110
|
+
|
111
|
+
res_match = re.search(res_pattern, filter_visible_chars(new_packet['data']))
|
112
|
+
if res_match is None:
|
113
|
+
req_len += len(new_packet['data'])
|
85
114
|
else:
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
115
|
+
print("匹配到响应")
|
116
|
+
res_len += len(new_packet['data'])
|
117
|
+
# 判断新的包是不是第二个请求包
|
118
|
+
if re.search(req_pattern, new_packet['data'].decode("utf-8", errors="ignore")):
|
119
|
+
print("这个包是个新的请求包的开头,停止查找")
|
120
|
+
break
|
121
|
+
packet_data += new_packet['data']
|
122
|
+
request_time += new_packet['time']
|
123
|
+
next_ack = f"{new_packet['last_len'] + new_packet['last_seq']}"
|
124
|
+
map = {}
|
125
|
+
data = filter_visible_chars(packet_data)
|
126
|
+
match_req = re.search(
|
127
|
+
req_res_pattern,
|
128
|
+
data)
|
129
|
+
match_res = re.search(res_pattern, data)
|
130
|
+
map['data'] = packet_data
|
131
|
+
map['req_len'] = req_len
|
132
|
+
map['res_len'] = res_len
|
133
|
+
map['time'] = request_time
|
134
|
+
map['req'] = match_req.group() if match_req is not None else ""
|
135
|
+
map['res'] = match_res.group() if match_res is not None else ""
|
136
|
+
packet_list.append(map)
|
137
|
+
return packet_list
|
95
138
|
|
96
139
|
|
97
140
|
def get_body(param, is_src):
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|