xbase-util 0.5.0__tar.gz → 0.5.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. {xbase_util-0.5.0 → xbase_util-0.5.1}/PKG-INFO +1 -1
  2. {xbase_util-0.5.0 → xbase_util-0.5.1}/setup.py +1 -1
  3. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/packet_util.py +60 -78
  4. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util.egg-info/PKG-INFO +1 -1
  5. {xbase_util-0.5.0 → xbase_util-0.5.1}/README.md +0 -0
  6. {xbase_util-0.5.0 → xbase_util-0.5.1}/setup.cfg +0 -0
  7. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/__init__.py +0 -0
  8. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/add_column_util.py +0 -0
  9. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/dangerous_util.py +0 -0
  10. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/db/__init__.py +0 -0
  11. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/db/bean/ConfigBean.py +0 -0
  12. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/db/bean/CurrentConfigBean.py +0 -0
  13. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/db/bean/FlowBean.py +0 -0
  14. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/db/bean/TaskTemplateBean.py +0 -0
  15. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/db/bean/__init__.py +0 -0
  16. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/db/dao/ConfigDao.py +0 -0
  17. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/db/dao/CurrentConfigDao.py +0 -0
  18. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/db/dao/FlowDao.py +0 -0
  19. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/db/dao/TaskTemplateDao.py +0 -0
  20. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/db/dao/__init__.py +0 -0
  21. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/db/initsqlite3.py +0 -0
  22. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/es_db_util.py +0 -0
  23. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/esreq.py +0 -0
  24. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/geo_util.py +0 -0
  25. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/handle_features_util.py +0 -0
  26. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/pcap_util.py +0 -0
  27. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/xbase_constant.py +0 -0
  28. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util/xbase_util.py +0 -0
  29. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util.egg-info/SOURCES.txt +0 -0
  30. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util.egg-info/dependency_links.txt +0 -0
  31. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util.egg-info/not-zip-safe +0 -0
  32. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util.egg-info/top_level.txt +0 -0
  33. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util_assets/GeoLite2-City.mmdb +0 -0
  34. {xbase_util-0.5.0 → xbase_util-0.5.1}/xbase_util_assets/arkimeparse.js +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: xbase_util
3
- Version: 0.5.0
3
+ Version: 0.5.1
4
4
  Summary: 网络安全基础工具
5
5
  Home-page: https://gitee.com/jimonik/xbase_util.git
6
6
  Author: xyt
@@ -3,7 +3,7 @@ from distutils.core import setup
3
3
  from setuptools import find_packages
4
4
 
5
5
  setup(name="xbase_util",
6
- version="0.5.0",
6
+ version="0.5.1",
7
7
  description="网络安全基础工具",
8
8
  long_description="包含提取,预测,训练的基础工具",
9
9
  author="xyt",
@@ -1,6 +1,6 @@
1
1
  import re
2
2
 
3
- from scapy.layers.inet import TCP
3
+ from scapy.layers.inet import TCP, IP
4
4
  from scapy.packet import Raw
5
5
 
6
6
  from xbase_util.xbase_constant import plain_content_type_columns, packetKeyname, src_dst_header, statisticHeader, \
@@ -58,85 +58,67 @@ def get_all_columns(
58
58
  return result_columns
59
59
 
60
60
 
61
+ req_pattern = re.compile(r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?\r\n\r\n",
62
+ re.DOTALL)
63
+ res_pattern = re.compile(r"HTTP/\d\.\d \d{3}.*", re.DOTALL)
64
+ req_body_pattern = re.compile(
65
+ r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?(?=HTTP/\d\.\d)", re.DOTALL)
66
+
67
+
61
68
  def get_all_packets_by_reg(packets):
62
- req_pattern = re.compile(r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?\r\n\r\n",re.DOTALL)
63
- req_body_pattern = re.compile(
64
- r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?(?=HTTP/\d\.\d)",re.DOTALL)
65
- res_pattern = re.compile(r"HTTP/\d\.\d \d{3}.*",re.DOTALL)
66
- tcp_packet_map = {}
67
- for packet in packets:
68
- if packet.haslayer(TCP) and packet.haslayer(Raw):
69
- raw_data = bytes(packet[Raw].load)
70
- ack = f"{packet[TCP].ack}"
71
- seq = packet[TCP].seq
72
- time = packet[TCP].time
73
- if f"{packet[TCP].ack}" not in tcp_packet_map:
74
- tcp_packet_map[ack] = {
75
- "data": raw_data,
76
- "time": [time],
77
- "seq": [seq],
78
- "last_seq": seq,
79
- "ack": ack,
80
- "len": [len(raw_data)],
81
- "last_len": len(raw_data)
82
- }
83
- else:
84
- tcp_packet_map[ack]['data'] += raw_data
85
- tcp_packet_map[ack]['time'].append(time)
86
- tcp_packet_map[ack]['seq'].append(seq)
87
- tcp_packet_map[ack]['last_len'] = len(raw_data)
88
- tcp_packet_map[ack]['last_seq'] = seq
69
+ packets = [packet for packet in packets if packet.haslayer(TCP) and packet.haslayer(IP) and packet.haslayer(Raw)]
89
70
  packet_list = []
90
- for ack, data_set in tcp_packet_map.items():
91
- data_str = data_set['data'].decode("utf-8", errors="ignore")
92
- request_re = re.search(req_pattern, data_str)
93
- if request_re is None:
94
- continue
95
- next_ack = f"{data_set['last_len'] + data_set['last_seq']}"
96
- packet_data = data_set['data']
97
- request_time = data_set['time']
98
- response_time=[]
99
- req_len = len(packet_data)
100
- res_len = 0
101
- while True:
102
- # 持续往下一个包找,直到下一个包是请求为止,因为下一个包可能还是属于这个包的一部分,也可能是响应的一部分
103
- # 下一个包的ack存在
104
- if next_ack not in tcp_packet_map:
105
- print("没找到新的ack")
106
- break
107
- new_packet = tcp_packet_map[next_ack]
108
- # 判断新的包是不是响应包
109
- res_match = re.search(res_pattern, filter_visible_chars(new_packet['data']))
110
- if res_match is None:
111
- req_len += len(new_packet['data'])
112
- request_time += new_packet['time']
113
- else:
114
- print("匹配到响应")
115
- res_len += len(new_packet['data'])
116
- response_time += new_packet['time']
117
-
118
- # 判断新的包是不是第二个请求包
119
- if re.search(req_pattern, new_packet['data'].decode("utf-8", errors="ignore")):
120
- print("这个包是个新的请求包的开头,停止查找")
121
- break
122
- packet_data += new_packet['data']
123
- next_ack = f"{new_packet['last_len'] + new_packet['last_seq']}"
124
- map = {}
125
- data = filter_visible_chars(packet_data)
126
- match_req = re.search(
127
- req_body_pattern,
128
- data)
129
- match_res = re.search(res_pattern, data)
130
- map['data'] = packet_data
131
- map['req_len'] = req_len
132
- map['res_len'] = res_len
133
- map['request_time'] = request_time
134
- map['response_time'] = response_time
135
- map['req'] = match_req.group() if match_req is not None else ""
136
- map['res'] = match_res.group() if match_res is not None else ""
137
- packet_list.append(map)
71
+ my_map = {
72
+ 'req_data': b'',
73
+ 'res_data': b'',
74
+ 'req_text': '',
75
+ 'res_text': '',
76
+ 'req_time': [],
77
+ 'res_time': []
78
+ }
79
+ last_is_req = None
80
+ for item in packets:
81
+ data = item[Raw].load
82
+ time = item.time
83
+ req_match = req_pattern.search(filter_visible_chars(data))
84
+ res_match = res_pattern.search(filter_visible_chars(data))
85
+ if req_match is not None or res_match is not None:
86
+ if req_match:
87
+ # 新的请求:请求时间不为空或者响应时间不为空,说明不为空,添加到列表并清空数据
88
+ if len(my_map['req_time']) != 0 or len(my_map['res_time']) != 0:
89
+ packet_list.append(my_map.copy())
90
+ my_map = {
91
+ 'req_data': data,
92
+ 'res_data': b'',
93
+ 'req_text': filter_visible_chars(data),
94
+ 'res_text': '',
95
+ 'req_time': [time],
96
+ 'res_time': []
97
+ }
98
+ last_is_req = True
99
+ if res_match:
100
+ my_map['res_data'] += data
101
+ my_map['res_text'] = filter_visible_chars(my_map['res_data'])
102
+ my_map['res_time'].append(time)
103
+ last_is_req = False
104
+ else:
105
+ # 不是请求不是相应,就是中间的包
106
+ if last_is_req is None:
107
+ # 一开始就没匹配到请求或者响应头,那就不管即使是中间的包
108
+ continue
109
+ if last_is_req is True:
110
+ my_map['req_time'].append(time)
111
+ my_map['req_data'] += data
112
+ my_map['req_text'] = filter_visible_chars(my_map['req_data'])
113
+ elif last_is_req is False:
114
+ my_map['res_time'].append(time)
115
+ my_map['res_data'] += data
116
+ my_map['res_text'] = filter_visible_chars(my_map['res_data'])
117
+ if len(my_map['req_time']) != 0 or len(my_map['res_time']) != 0:
118
+ packet_list.append(my_map.copy())
138
119
  return packet_list
139
120
 
121
+
140
122
  def get_body(param):
141
123
  body = "".join([item.strip() for item in param.split("\r\n\r\n") if item.strip() != "" and "HTTP/" not in param])
142
124
  return "" if body is None else body
@@ -159,8 +141,8 @@ def get_detail_by_package(packets_from_pcap, publicField, use_regx):
159
141
  """
160
142
  res_field = publicField.copy()
161
143
  if use_regx:
162
- req = packets_from_pcap['req']
163
- res = packets_from_pcap['res']
144
+ req = packets_from_pcap['req_text']
145
+ res = packets_from_pcap['res_text']
164
146
  else:
165
147
  res = packets_from_pcap["response"]
166
148
  req = packets_from_pcap["request"]
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: xbase-util
3
- Version: 0.5.0
3
+ Version: 0.5.1
4
4
  Summary: 网络安全基础工具
5
5
  Home-page: https://gitee.com/jimonik/xbase_util.git
6
6
  Author: xyt
File without changes
File without changes