xbase-util 0.5.0__tar.gz → 0.5.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {xbase_util-0.5.0 → xbase_util-0.5.1}/PKG-INFO +1 -1
- {xbase_util-0.5.0 → xbase_util-0.5.1}/setup.py +1 -1
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/packet_util.py +60 -78
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util.egg-info/PKG-INFO +1 -1
- {xbase_util-0.5.0 → xbase_util-0.5.1}/README.md +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/setup.cfg +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/__init__.py +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/add_column_util.py +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/dangerous_util.py +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/db/__init__.py +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/db/bean/ConfigBean.py +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/db/bean/CurrentConfigBean.py +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/db/bean/FlowBean.py +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/db/bean/TaskTemplateBean.py +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/db/bean/__init__.py +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/db/dao/ConfigDao.py +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/db/dao/CurrentConfigDao.py +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/db/dao/FlowDao.py +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/db/dao/TaskTemplateDao.py +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/db/dao/__init__.py +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/db/initsqlite3.py +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/es_db_util.py +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/esreq.py +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/geo_util.py +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/handle_features_util.py +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/pcap_util.py +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/xbase_constant.py +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/xbase_util.py +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util.egg-info/SOURCES.txt +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util.egg-info/dependency_links.txt +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util.egg-info/not-zip-safe +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util.egg-info/top_level.txt +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util_assets/GeoLite2-City.mmdb +0 -0
- {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util_assets/arkimeparse.js +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
import re
|
2
2
|
|
3
|
-
from scapy.layers.inet import TCP
|
3
|
+
from scapy.layers.inet import TCP, IP
|
4
4
|
from scapy.packet import Raw
|
5
5
|
|
6
6
|
from xbase_util.xbase_constant import plain_content_type_columns, packetKeyname, src_dst_header, statisticHeader, \
|
@@ -58,85 +58,67 @@ def get_all_columns(
|
|
58
58
|
return result_columns
|
59
59
|
|
60
60
|
|
61
|
+
req_pattern = re.compile(r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?\r\n\r\n",
|
62
|
+
re.DOTALL)
|
63
|
+
res_pattern = re.compile(r"HTTP/\d\.\d \d{3}.*", re.DOTALL)
|
64
|
+
req_body_pattern = re.compile(
|
65
|
+
r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?(?=HTTP/\d\.\d)", re.DOTALL)
|
66
|
+
|
67
|
+
|
61
68
|
def get_all_packets_by_reg(packets):
|
62
|
-
|
63
|
-
req_body_pattern = re.compile(
|
64
|
-
r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?(?=HTTP/\d\.\d)",re.DOTALL)
|
65
|
-
res_pattern = re.compile(r"HTTP/\d\.\d \d{3}.*",re.DOTALL)
|
66
|
-
tcp_packet_map = {}
|
67
|
-
for packet in packets:
|
68
|
-
if packet.haslayer(TCP) and packet.haslayer(Raw):
|
69
|
-
raw_data = bytes(packet[Raw].load)
|
70
|
-
ack = f"{packet[TCP].ack}"
|
71
|
-
seq = packet[TCP].seq
|
72
|
-
time = packet[TCP].time
|
73
|
-
if f"{packet[TCP].ack}" not in tcp_packet_map:
|
74
|
-
tcp_packet_map[ack] = {
|
75
|
-
"data": raw_data,
|
76
|
-
"time": [time],
|
77
|
-
"seq": [seq],
|
78
|
-
"last_seq": seq,
|
79
|
-
"ack": ack,
|
80
|
-
"len": [len(raw_data)],
|
81
|
-
"last_len": len(raw_data)
|
82
|
-
}
|
83
|
-
else:
|
84
|
-
tcp_packet_map[ack]['data'] += raw_data
|
85
|
-
tcp_packet_map[ack]['time'].append(time)
|
86
|
-
tcp_packet_map[ack]['seq'].append(seq)
|
87
|
-
tcp_packet_map[ack]['last_len'] = len(raw_data)
|
88
|
-
tcp_packet_map[ack]['last_seq'] = seq
|
69
|
+
packets = [packet for packet in packets if packet.haslayer(TCP) and packet.haslayer(IP) and packet.haslayer(Raw)]
|
89
70
|
packet_list = []
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
packet_list.append(
|
71
|
+
my_map = {
|
72
|
+
'req_data': b'',
|
73
|
+
'res_data': b'',
|
74
|
+
'req_text': '',
|
75
|
+
'res_text': '',
|
76
|
+
'req_time': [],
|
77
|
+
'res_time': []
|
78
|
+
}
|
79
|
+
last_is_req = None
|
80
|
+
for item in packets:
|
81
|
+
data = item[Raw].load
|
82
|
+
time = item.time
|
83
|
+
req_match = req_pattern.search(filter_visible_chars(data))
|
84
|
+
res_match = res_pattern.search(filter_visible_chars(data))
|
85
|
+
if req_match is not None or res_match is not None:
|
86
|
+
if req_match:
|
87
|
+
# 新的请求:请求时间不为空或者响应时间不为空,说明不为空,添加到列表并清空数据
|
88
|
+
if len(my_map['req_time']) != 0 or len(my_map['res_time']) != 0:
|
89
|
+
packet_list.append(my_map.copy())
|
90
|
+
my_map = {
|
91
|
+
'req_data': data,
|
92
|
+
'res_data': b'',
|
93
|
+
'req_text': filter_visible_chars(data),
|
94
|
+
'res_text': '',
|
95
|
+
'req_time': [time],
|
96
|
+
'res_time': []
|
97
|
+
}
|
98
|
+
last_is_req = True
|
99
|
+
if res_match:
|
100
|
+
my_map['res_data'] += data
|
101
|
+
my_map['res_text'] = filter_visible_chars(my_map['res_data'])
|
102
|
+
my_map['res_time'].append(time)
|
103
|
+
last_is_req = False
|
104
|
+
else:
|
105
|
+
# 不是请求不是相应,就是中间的包
|
106
|
+
if last_is_req is None:
|
107
|
+
# 一开始就没匹配到请求或者响应头,那就不管即使是中间的包
|
108
|
+
continue
|
109
|
+
if last_is_req is True:
|
110
|
+
my_map['req_time'].append(time)
|
111
|
+
my_map['req_data'] += data
|
112
|
+
my_map['req_text'] = filter_visible_chars(my_map['req_data'])
|
113
|
+
elif last_is_req is False:
|
114
|
+
my_map['res_time'].append(time)
|
115
|
+
my_map['res_data'] += data
|
116
|
+
my_map['res_text'] = filter_visible_chars(my_map['res_data'])
|
117
|
+
if len(my_map['req_time']) != 0 or len(my_map['res_time']) != 0:
|
118
|
+
packet_list.append(my_map.copy())
|
138
119
|
return packet_list
|
139
120
|
|
121
|
+
|
140
122
|
def get_body(param):
|
141
123
|
body = "".join([item.strip() for item in param.split("\r\n\r\n") if item.strip() != "" and "HTTP/" not in param])
|
142
124
|
return "" if body is None else body
|
@@ -159,8 +141,8 @@ def get_detail_by_package(packets_from_pcap, publicField, use_regx):
|
|
159
141
|
"""
|
160
142
|
res_field = publicField.copy()
|
161
143
|
if use_regx:
|
162
|
-
req = packets_from_pcap['
|
163
|
-
res = packets_from_pcap['
|
144
|
+
req = packets_from_pcap['req_text']
|
145
|
+
res = packets_from_pcap['res_text']
|
164
146
|
else:
|
165
147
|
res = packets_from_pcap["response"]
|
166
148
|
req = packets_from_pcap["request"]
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|