xbase-util 0.0.7__tar.gz → 0.0.8__tar.gz

Sign up to get free protection for your applications and to get access to all the features.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: xbase_util
3
- Version: 0.0.7
3
+ Version: 0.0.8
4
4
  Summary: 网络安全基础工具
5
5
  Author: xyt
6
6
  Author-email: 2506564278@qq.com
@@ -3,7 +3,7 @@ from distutils.core import setup
3
3
  from setuptools import find_packages
4
4
 
5
5
  setup(name="xbase_util",
6
- version="0.0.7",
6
+ version="0.0.8",
7
7
  description="网络安全基础工具",
8
8
  long_description="包含提取,预测,训练的基础工具",
9
9
  author="xyt",
@@ -0,0 +1,86 @@
1
+ import os
2
+ import re
3
+
4
+ import execjs
5
+ import geoip2.database
6
+
7
+ current_dir = os.path.dirname(__file__)
8
+ parse_path = os.path.join(current_dir, '..', 'xbase_util_assets', 'arkimeparse.js')
9
+ geo_path = os.path.join(current_dir, '..', 'xbase_util_assets', 'GeoLite2-City.mmdb')
10
+
11
+
12
+ def parse_expression(expression):
13
+ if expression:
14
+ with open(parse_path, "r") as f:
15
+ ctx = execjs.compile(f.read())
16
+ return ctx.call("parse_exp", expression)
17
+ else:
18
+ return None
19
+
20
+
21
+ def geo_reader():
22
+ return geoip2.database.Reader(geo_path)
23
+
24
+
25
+ def split_samples(sample, per_subsection):
26
+ num_subsections = len(sample) // per_subsection
27
+ remainder = len(sample) % per_subsection
28
+ subsection_sizes = [per_subsection] * num_subsections
29
+ if remainder > 0:
30
+ subsection_sizes.append(remainder)
31
+ num_subsections += 1
32
+ return num_subsections, subsection_sizes
33
+
34
+
35
+ def split_process(subsection, process_count):
36
+ subsection_per_process = len(subsection) // process_count
37
+ remainder = len(subsection) % process_count
38
+ lengths = []
39
+ start = 0
40
+ for i in range(process_count):
41
+ end = start + subsection_per_process + (1 if i < remainder else 0)
42
+ lengths.append(end - start)
43
+ start = end
44
+ return lengths
45
+
46
+
47
+ def build_es_expression(size, start_time, end_time, arkime_expression):
48
+ expression = {"query": {"bool": {"filter": []}}}
49
+ try:
50
+ if size:
51
+ expression['size'] = size
52
+ if start_time:
53
+ expression['query']['bool']['filter'].append(
54
+ {"range": {"firstPacket": {"gte": round(start_time.timestamp() * 1000)}}})
55
+ if end_time:
56
+ expression['query']['bool']['filter'].append(
57
+ {"range": {"lastPacket": {"lte": round(end_time.timestamp() * 1000)}}})
58
+ arkime_2_es = parse_expression(arkime_expression)
59
+ if arkime_2_es:
60
+ expression['query']['bool']['filter'].append(arkime_2_es)
61
+ return expression
62
+ except Exception as e:
63
+ print(f"请安装nodejs{e}")
64
+ print(arkime_expression)
65
+ exit(1)
66
+
67
+
68
+ def get_uri_depth(url):
69
+ match = re.match(r'^[^?]*', url)
70
+ if match:
71
+ path = match.group(0)
72
+ # 去除协议和域名部分
73
+ path = re.sub(r'^https?://[^/]+', '', path)
74
+ segments = [segment for segment in path.split('/') if segment]
75
+ return len(segments)
76
+ return 0
77
+
78
+
79
+ def firstOrZero(param):
80
+ if type(param).__name__ == 'list':
81
+ if (len(param)) != 0:
82
+ return param[0]
83
+ else:
84
+ return 0
85
+ else:
86
+ return 0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: xbase-util
3
- Version: 0.0.7
3
+ Version: 0.0.8
4
4
  Summary: 网络安全基础工具
5
5
  Author: xyt
6
6
  Author-email: 2506564278@qq.com
@@ -1,21 +0,0 @@
1
- import os
2
-
3
- import execjs
4
- import geoip2.database
5
-
6
- current_dir = os.path.dirname(__file__)
7
- parse_path = os.path.join(current_dir, '..', 'xbase_util_assets', 'arkimeparse.js')
8
- geo_path = os.path.join(current_dir, '..', 'xbase_util_assets', 'GeoLite2-City.mmdb')
9
-
10
-
11
- def parse_expression(expression):
12
- if expression:
13
- with open(parse_path, "r") as f:
14
- ctx = execjs.compile(f.read())
15
- return ctx.call("parse_exp", expression)
16
- else:
17
- return None
18
-
19
-
20
- def geo_reader():
21
- return geoip2.database.Reader(geo_path)
File without changes
File without changes