xbase-util 0.0.7__tar.gz → 0.0.8__tar.gz
Sign up to get free protection for your applications and to get access to all the features.
- {xbase_util-0.0.7 → xbase_util-0.0.8}/PKG-INFO +1 -1
- {xbase_util-0.0.7 → xbase_util-0.0.8}/setup.py +1 -1
- xbase_util-0.0.8/xbase_util/xbase_util.py +86 -0
- {xbase_util-0.0.7 → xbase_util-0.0.8}/xbase_util.egg-info/PKG-INFO +1 -1
- xbase_util-0.0.7/xbase_util/xbase_util.py +0 -21
- {xbase_util-0.0.7 → xbase_util-0.0.8}/README.md +0 -0
- {xbase_util-0.0.7 → xbase_util-0.0.8}/setup.cfg +0 -0
- {xbase_util-0.0.7 → xbase_util-0.0.8}/xbase_util/__init__.py +0 -0
- {xbase_util-0.0.7 → xbase_util-0.0.8}/xbase_util.egg-info/SOURCES.txt +0 -0
- {xbase_util-0.0.7 → xbase_util-0.0.8}/xbase_util.egg-info/dependency_links.txt +0 -0
- {xbase_util-0.0.7 → xbase_util-0.0.8}/xbase_util.egg-info/not-zip-safe +0 -0
- {xbase_util-0.0.7 → xbase_util-0.0.8}/xbase_util.egg-info/top_level.txt +0 -0
- {xbase_util-0.0.7 → xbase_util-0.0.8}/xbase_util_assets/GeoLite2-City.mmdb +0 -0
- {xbase_util-0.0.7 → xbase_util-0.0.8}/xbase_util_assets/arkimeparse.js +0 -0
@@ -0,0 +1,86 @@
|
|
1
|
+
import os
|
2
|
+
import re
|
3
|
+
|
4
|
+
import execjs
|
5
|
+
import geoip2.database
|
6
|
+
|
7
|
+
current_dir = os.path.dirname(__file__)
|
8
|
+
parse_path = os.path.join(current_dir, '..', 'xbase_util_assets', 'arkimeparse.js')
|
9
|
+
geo_path = os.path.join(current_dir, '..', 'xbase_util_assets', 'GeoLite2-City.mmdb')
|
10
|
+
|
11
|
+
|
12
|
+
def parse_expression(expression):
|
13
|
+
if expression:
|
14
|
+
with open(parse_path, "r") as f:
|
15
|
+
ctx = execjs.compile(f.read())
|
16
|
+
return ctx.call("parse_exp", expression)
|
17
|
+
else:
|
18
|
+
return None
|
19
|
+
|
20
|
+
|
21
|
+
def geo_reader():
|
22
|
+
return geoip2.database.Reader(geo_path)
|
23
|
+
|
24
|
+
|
25
|
+
def split_samples(sample, per_subsection):
|
26
|
+
num_subsections = len(sample) // per_subsection
|
27
|
+
remainder = len(sample) % per_subsection
|
28
|
+
subsection_sizes = [per_subsection] * num_subsections
|
29
|
+
if remainder > 0:
|
30
|
+
subsection_sizes.append(remainder)
|
31
|
+
num_subsections += 1
|
32
|
+
return num_subsections, subsection_sizes
|
33
|
+
|
34
|
+
|
35
|
+
def split_process(subsection, process_count):
|
36
|
+
subsection_per_process = len(subsection) // process_count
|
37
|
+
remainder = len(subsection) % process_count
|
38
|
+
lengths = []
|
39
|
+
start = 0
|
40
|
+
for i in range(process_count):
|
41
|
+
end = start + subsection_per_process + (1 if i < remainder else 0)
|
42
|
+
lengths.append(end - start)
|
43
|
+
start = end
|
44
|
+
return lengths
|
45
|
+
|
46
|
+
|
47
|
+
def build_es_expression(size, start_time, end_time, arkime_expression):
|
48
|
+
expression = {"query": {"bool": {"filter": []}}}
|
49
|
+
try:
|
50
|
+
if size:
|
51
|
+
expression['size'] = size
|
52
|
+
if start_time:
|
53
|
+
expression['query']['bool']['filter'].append(
|
54
|
+
{"range": {"firstPacket": {"gte": round(start_time.timestamp() * 1000)}}})
|
55
|
+
if end_time:
|
56
|
+
expression['query']['bool']['filter'].append(
|
57
|
+
{"range": {"lastPacket": {"lte": round(end_time.timestamp() * 1000)}}})
|
58
|
+
arkime_2_es = parse_expression(arkime_expression)
|
59
|
+
if arkime_2_es:
|
60
|
+
expression['query']['bool']['filter'].append(arkime_2_es)
|
61
|
+
return expression
|
62
|
+
except Exception as e:
|
63
|
+
print(f"请安装nodejs{e}")
|
64
|
+
print(arkime_expression)
|
65
|
+
exit(1)
|
66
|
+
|
67
|
+
|
68
|
+
def get_uri_depth(url):
|
69
|
+
match = re.match(r'^[^?]*', url)
|
70
|
+
if match:
|
71
|
+
path = match.group(0)
|
72
|
+
# 去除协议和域名部分
|
73
|
+
path = re.sub(r'^https?://[^/]+', '', path)
|
74
|
+
segments = [segment for segment in path.split('/') if segment]
|
75
|
+
return len(segments)
|
76
|
+
return 0
|
77
|
+
|
78
|
+
|
79
|
+
def firstOrZero(param):
|
80
|
+
if type(param).__name__ == 'list':
|
81
|
+
if (len(param)) != 0:
|
82
|
+
return param[0]
|
83
|
+
else:
|
84
|
+
return 0
|
85
|
+
else:
|
86
|
+
return 0
|
@@ -1,21 +0,0 @@
|
|
1
|
-
import os
|
2
|
-
|
3
|
-
import execjs
|
4
|
-
import geoip2.database
|
5
|
-
|
6
|
-
current_dir = os.path.dirname(__file__)
|
7
|
-
parse_path = os.path.join(current_dir, '..', 'xbase_util_assets', 'arkimeparse.js')
|
8
|
-
geo_path = os.path.join(current_dir, '..', 'xbase_util_assets', 'GeoLite2-City.mmdb')
|
9
|
-
|
10
|
-
|
11
|
-
def parse_expression(expression):
|
12
|
-
if expression:
|
13
|
-
with open(parse_path, "r") as f:
|
14
|
-
ctx = execjs.compile(f.read())
|
15
|
-
return ctx.call("parse_exp", expression)
|
16
|
-
else:
|
17
|
-
return None
|
18
|
-
|
19
|
-
|
20
|
-
def geo_reader():
|
21
|
-
return geoip2.database.Reader(geo_path)
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|