xbase-util 0.4.6__tar.gz → 0.4.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. {xbase_util-0.4.6 → xbase_util-0.4.7}/PKG-INFO +1 -1
  2. {xbase_util-0.4.6 → xbase_util-0.4.7}/setup.py +1 -1
  3. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/packet_util.py +77 -34
  4. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util.egg-info/PKG-INFO +1 -1
  5. {xbase_util-0.4.6 → xbase_util-0.4.7}/README.md +0 -0
  6. {xbase_util-0.4.6 → xbase_util-0.4.7}/setup.cfg +0 -0
  7. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/__init__.py +0 -0
  8. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/add_column_util.py +0 -0
  9. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/dangerous_util.py +0 -0
  10. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/db/__init__.py +0 -0
  11. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/db/bean/ConfigBean.py +0 -0
  12. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/db/bean/CurrentConfigBean.py +0 -0
  13. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/db/bean/FlowBean.py +0 -0
  14. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/db/bean/TaskTemplateBean.py +0 -0
  15. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/db/bean/__init__.py +0 -0
  16. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/db/dao/ConfigDao.py +0 -0
  17. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/db/dao/CurrentConfigDao.py +0 -0
  18. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/db/dao/FlowDao.py +0 -0
  19. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/db/dao/TaskTemplateDao.py +0 -0
  20. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/db/dao/__init__.py +0 -0
  21. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/db/initsqlite3.py +0 -0
  22. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/es_db_util.py +0 -0
  23. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/esreq.py +0 -0
  24. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/geo_util.py +0 -0
  25. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/handle_features_util.py +0 -0
  26. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/pcap_util.py +0 -0
  27. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/xbase_constant.py +0 -0
  28. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util/xbase_util.py +0 -0
  29. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util.egg-info/SOURCES.txt +0 -0
  30. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util.egg-info/dependency_links.txt +0 -0
  31. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util.egg-info/not-zip-safe +0 -0
  32. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util.egg-info/top_level.txt +0 -0
  33. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util_assets/GeoLite2-City.mmdb +0 -0
  34. {xbase_util-0.4.6 → xbase_util-0.4.7}/xbase_util_assets/arkimeparse.js +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: xbase_util
3
- Version: 0.4.6
3
+ Version: 0.4.7
4
4
  Summary: 网络安全基础工具
5
5
  Home-page: https://gitee.com/jimonik/xbase_util.git
6
6
  Author: xyt
@@ -3,7 +3,7 @@ from distutils.core import setup
3
3
  from setuptools import find_packages
4
4
 
5
5
  setup(name="xbase_util",
6
- version="0.4.6",
6
+ version="0.4.7",
7
7
  description="网络安全基础工具",
8
8
  long_description="包含提取,预测,训练的基础工具",
9
9
  author="xyt",
@@ -1,6 +1,7 @@
1
1
  import re
2
2
 
3
3
  from scapy.layers.inet import TCP
4
+ from scapy.packet import Raw
4
5
 
5
6
  from xbase_util.xbase_constant import plain_content_type_columns, packetKeyname, src_dst_header, statisticHeader, \
6
7
  features_key, plain_body_columns
@@ -57,41 +58,83 @@ def get_all_columns(
57
58
  return result_columns
58
59
 
59
60
 
60
- def get_all_packets_by_regx(packets):
61
- """
62
- 通过正则pcap获取所有包的数据
63
- :param packets:
64
- :return:
65
- """
66
- streams = b""
67
- for pkt in packets:
68
- if TCP in pkt:
69
- streams += bytes(pkt[TCP].payload)
70
- text = filter_visible_chars(streams)
71
- pattern = r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d"
72
- requests = re.split(f"(?={pattern})", text, re.M)
73
- all_packets = []
74
- for item in requests:
75
- if len(re.findall(pattern, item)) != 0:
76
- request_text = ""
77
- response_text = ""
78
- response_text_list = re.findall(r"HTTP/\d\.\d \d{3}[\s\S]*", item)
79
- if len(response_text_list) != 0:
80
- # 有响应数据
81
- response_text = response_text_list[0]
82
- if response_text == "":
83
- # 没有响应数据,那么全是请求数据
84
- request_text = item
61
+ def get_all_packets_by_reg(packets):
62
+ req_pattern = re.compile(r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?\r\n\r\n",
63
+ re.DOTALL)
64
+ req_res_pattern = re.compile(
65
+ r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?(?=HTTP/\d\.\d \d{3} [a-zA-Z]+|$)",
66
+ re.DOTALL)
67
+ res_pattern = re.compile(r"HTTP/\d\.\d \d{3} [a-zA-Z]+.*", re.DOTALL)
68
+ tcp_packet_map = {}
69
+ for packet in packets:
70
+ if packet.haslayer(TCP) and packet.haslayer(Raw):
71
+ raw_data = bytes(packet[Raw].load)
72
+ ack = f"{packet[TCP].ack}"
73
+ seq = packet[TCP].seq
74
+ time = packet[TCP].time
75
+ if f"{packet[TCP].ack}" not in tcp_packet_map:
76
+ tcp_packet_map[ack] = {
77
+ "data": raw_data,
78
+ "time": [time],
79
+ "seq": [seq],
80
+ "last_seq": seq,
81
+ "ack": ack,
82
+ "len": [len(raw_data)],
83
+ "last_len": len(raw_data)
84
+ }
85
+ else:
86
+ tcp_packet_map[ack]['data'] += raw_data
87
+ tcp_packet_map[ack]['time'].append(time)
88
+ tcp_packet_map[ack]['seq'].append(seq)
89
+ tcp_packet_map[ack]['last_len'] = len(raw_data)
90
+ tcp_packet_map[ack]['last_seq'] = seq
91
+ packet_list = []
92
+ for ack, data_set in tcp_packet_map.items():
93
+ data_str = data_set['data'].decode("utf-8", errors="ignore")
94
+ request_re = re.search(req_pattern, data_str)
95
+ if request_re is None:
96
+ continue
97
+ next_ack = f"{data_set['last_len'] + data_set['last_seq']}"
98
+ packet_data = data_set['data']
99
+ request_time = data_set['time']
100
+ req_len = len(packet_data)
101
+ res_len = 0
102
+ while True:
103
+ # 持续往下一个包找,直到下一个包是请求为止,因为下一个包可能还是属于这个包的一部分,也可能是响应的一部分
104
+ # 下一个包的ack存在
105
+ if next_ack not in tcp_packet_map:
106
+ print("没找到新的ack")
107
+ break
108
+ new_packet = tcp_packet_map[next_ack]
109
+ # 判断新的包是不是响应包
110
+
111
+ res_match = re.search(res_pattern, filter_visible_chars(new_packet['data']))
112
+ if res_match is None:
113
+ req_len += len(new_packet['data'])
85
114
  else:
86
- # 有响应数据,用正则获取请求数据
87
- request_re = re.search(
88
- r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?\r\n\r\n", item)
89
- if request_re:
90
- request_text = request_re.group(0)
91
- else:
92
- request_text = ""
93
- all_packets.append({"req": request_text, "res": response_text})
94
- return all_packets
115
+ print("匹配到响应")
116
+ res_len += len(new_packet['data'])
117
+ # 判断新的包是不是第二个请求包
118
+ if re.search(req_pattern, new_packet['data'].decode("utf-8", errors="ignore")):
119
+ print("这个包是个新的请求包的开头,停止查找")
120
+ break
121
+ packet_data += new_packet['data']
122
+ request_time += new_packet['time']
123
+ next_ack = f"{new_packet['last_len'] + new_packet['last_seq']}"
124
+ map = {}
125
+ data = filter_visible_chars(packet_data)
126
+ match_req = re.search(
127
+ req_res_pattern,
128
+ data)
129
+ match_res = re.search(res_pattern, data)
130
+ map['data'] = packet_data
131
+ map['req_len'] = req_len
132
+ map['res_len'] = res_len
133
+ map['time'] = request_time
134
+ map['req'] = match_req.group() if match_req is not None else ""
135
+ map['res'] = match_res.group() if match_res is not None else ""
136
+ packet_list.append(map)
137
+ return packet_list
95
138
 
96
139
 
97
140
  def get_body(param, is_src):
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: xbase-util
3
- Version: 0.4.6
3
+ Version: 0.4.7
4
4
  Summary: 网络安全基础工具
5
5
  Home-page: https://gitee.com/jimonik/xbase_util.git
6
6
  Author: xyt
File without changes
File without changes