xbase-util 0.4.2__tar.gz → 0.4.4__tar.gz

{xbase_util-0.4.2 → xbase_util-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase_util
-Version: 0.4.2
+Version: 0.4.4
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt

{xbase_util-0.4.2 → xbase_util-0.4.4}/setup.py

@@ -3,7 +3,7 @@ from distutils.core import setup
 from setuptools import find_packages
 
 setup(name="xbase_util",
-      version="0.4.2",
+      version="0.4.4",
       description="网络安全基础工具",
       long_description="包含提取，预测，训练的基础工具",
       author="xyt",

xbase_util-0.4.4/xbase_util/dangerous_util.py

@@ -0,0 +1,101 @@
+import splunklib.client as client
+from splunklib import results
+
+
+def get_splunk_pa(start_time, end_time, splunk_host,
+                  splunk_port,
+                  splunk_username,
+                  splunk_password,
+                  splunk_scheme="https",
+                  splunk_filter="THREAT AND NOT informational"):
+    """
+    获取PA威胁信息
+    :param splunk_filter:
+    :param start_time:
+    :param end_time:
+    :param splunk_host:
+    :param splunk_port:
+    :param splunk_username:
+    :param splunk_password:
+    :param splunk_scheme:
+    :return:
+    """
+    service = client.connect(
+        host=splunk_host,
+        port=splunk_port,
+        scheme=splunk_scheme,
+        username=splunk_username,
+        password=splunk_password
+    )
+    job = service.jobs.oneshot(
+        """search index=idx_pa 
+FILTER_TEXT
+| rex field=_raw  "THREAT,(?P<LOG_TYPE>.+?),.*?,(?P<PA_DATE>.*?),(?P<SIP>.*?),(?P<DIP>.*?),(?:.*?,.*?,.*?){7},(?P<S_PORT>.*?),(?P<D_PORT>.*?),.*?,.*?,.*?,(?P<PROTOCOL>.*?),(?P<DENY_METHOD>.*?),(?P<THREAT_SUMMARY>.*?),(?P<SEVERITY>medium|high|critical|low),"  
+| eval RAW_BAK=_raw 
+| eval THREAT_TIME = strftime(strptime(PA_DATE, "%Y/%m/%d %H:%M:%S"), "%Y-%m-%d %H:%M:%S")
+| rex mode=sed field=_raw "s/.*?,\w{8}-\w{4}-\w{4}-\w{4}-\w{12},/start_ama_flag,/g"
+| rex field=_raw "start_ama_flag,.*?,.*?,(?<XFF_IP>.*?),"
+| eval _raw=RAW_BAK
+| table THREAT_TIME,SIP,S_PORT, DIP, D_PORT,XFF_IP,PROTOCOL, DENY_METHOD, THREAT_SUMMARY, SEVERITY
+| dedup THREAT_TIME,SIP,S_PORT, DIP, D_PORT,XFF_IP,PROTOCOL
+            """.replace("FILTER_TEXT", splunk_filter), **{
+            "earliest_time": start_time.strftime('%Y-%m-%dT%H:%M:%S'),
+            "latest_time": end_time.strftime('%Y-%m-%dT%H:%M:%S'),
+            "output_mode": "json",
+            "count": 20000
+        })
+    return [item for item in results.JSONResultsReader(job) if isinstance(item, dict)]
+
+
+def get_splunk_waf(start_time, end_time, splunk_host,
+                   splunk_port,
+                   splunk_username,
+                   splunk_password,
+                   splunk_scheme="https"):
+    # splunk里面用这个
+    """
+sourcetype=changting:waf
+| rex field=_raw (?P<THREAT_TIME>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+\d{2}:\d{2})
+| rex field=_raw "\"src_ip\":\"(?<SIP>[^\"]+)\""
+| rex field=_raw "\"protocol\":\"(?<PROTOCOL>[^\"]+)\""
+| rex field=_raw "\"src_port\":(?<S_PORT>\d+)"
+| rex field=_raw "\"dest_ip\":\"(?<DIP>[^\"]+)\""
+| rex field=_raw "\"dest_port\":(?<D_PORT>\d+)"
+| rex field=_raw "\"risk_level\":\"(?<SEVERITY>[^\"]+)\""
+| rex field=_raw "\"action\":\"(?<DENY_METHOD>[^\"]+)\""
+| rex field=_raw "\"reason\":\"(?<THREAT_SUMMARY>[^\"]+)\""
+| rex field=_raw "\"x_forwarded_for\":\"(?<XFF_IP>[^\"]+)\""
+| table THREAT_TIME,SIP,S_PORT,DIP,D_PORT,XFF_IP,PROTOCOL,DENY_METHOD,THREAT_SUMMARY,SEVERITY
+| dedup THREAT_TIME,SIP,S_PORT,DIP,D_PORT,XFF_IP,PROTOCOL
+    """
+    service = client.connect(
+        host=splunk_host,
+        port=splunk_port,
+        scheme=splunk_scheme,
+        username=splunk_username,
+        password=splunk_password
+    )
+    # | rex field=_raw "\\"src_port\\":(?<S_PORT>\d+)"
+    # | rex field=_raw "\\"dest_port\\":(?<D_PORT>\d+)"
+    exp = """search sourcetype=changting:waf
+| rex field=_raw "(?P<THREAT_TIME>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\+\\d{2}:\\d{2})"
+| rex field=_raw "\\"dest_ip\\":\\"(?<DIP>[^\\"]+)\\"" 
+| rex field=_raw "\\"src_ip\\":\\"(?<SIP>[^\\"]+)\\""
+| rex field=_raw "\\"src_port\\":(?<S_PORT>\\d+)"
+| rex field=_raw "\\"dest_port\\":(?<D_PORT>\\d+)"
+| rex field=_raw "\\"protocol\\":\\"(?<PROTOCOL>[^\\"]+)\\""
+| rex field=_raw "\\"x_forwarded_for\\":\\"(?<XFF_IP>[^\\"]+)\\""
+| rex field=_raw "\\"action\\":\\"(?<DENY_METHOD>[^\\"]+)\\""
+| rex field=_raw "\\"reason\\":\\"(?<THREAT_SUMMARY>[^\\"]+)\\""
+| rex field=_raw "\\"risk_level\\":\\"(?<SEVERITY>[^\\"]+)\\""
+| dedup THREAT_TIME,SIP,S_PORT,DIP,D_PORT,XFF_IP,PROTOCOL
+| table THREAT_TIME,SIP,S_PORT,DIP,D_PORT,XFF_IP,PROTOCOL,DENY_METHOD,THREAT_SUMMARY,SEVERITY
+        """
+    job = service.jobs.oneshot(
+        exp, **{
+            "earliest_time": start_time.strftime('%Y-%m-%dT%H:%M:%S'),
+            "latest_time": end_time.strftime('%Y-%m-%dT%H:%M:%S'),
+            "output_mode": "json",
+            "count": 20000
+        })
+    return [item for item in results.JSONResultsReader(job) if isinstance(item, dict)]

{xbase_util-0.4.2 → xbase_util-0.4.4}/xbase_util/es_db_util.py

@@ -5,7 +5,6 @@ class EsDb:
     def __init__(self, req, manager):
         self.req = req
         self.internals = manager.dict()
-        print("初始化:Elasticsearch DB")
 
     def get_file_by_file_id(self, node, num, prefix=None):
         key = f'{node}!{num}'

xbase_util-0.4.4/xbase_util/packet_util.py

@@ -0,0 +1,171 @@
+import re
+
+from scapy.layers.inet import TCP
+
+from xbase_util.xbase_constant import plain_content_type_columns, packetKeyname, src_dst_header, statisticHeader, \
+    features_key, plain_body_columns
+from xbase_util.xbase_util import firstOrZero
+
+
+def content_type_is_plain(packet):
+    """
+    从单个包（包括header和body）中获取content-type并判断是否为可见类型
+    :param packet:
+    :return:
+    """
+    if ":" not in packet:
+        return False
+    for item in packet.replace("-", "_").replace(" ", "").lower().split("\n"):
+        if "content_type" in item:
+            if ":" not in item:
+                continue
+            content_type = item.split(":")[1].replace("\r", "").strip()
+            return content_type in plain_content_type_columns
+    return False
+
+
+def filter_visible_chars(data):
+    """
+    过滤不可见字符，仅保留可打印的ASCII字符
+    :param data:
+    :return:
+    """
+    return ''.join(chr(b) for b in data if 32 <= b <= 126 or b in (9, 10, 13))
+
+
+def get_all_columns(
+        contains_packet_column=False,
+        contains_src_dst_column=False,
+        contains_statistic_column=False,
+        contains_features_column=False,
+        contains_plain_body_column=False,
+        contains_pcap_flow_text=False
+):
+    result_columns = []
+    if contains_packet_column:
+        result_columns += packetKeyname
+    if contains_src_dst_column:
+        result_columns += src_dst_header
+    if contains_statistic_column:
+        result_columns += statisticHeader
+    if contains_features_column:
+        result_columns += features_key
+    if contains_plain_body_column:
+        result_columns += plain_body_columns
+    if contains_pcap_flow_text:
+        result_columns.append(contains_pcap_flow_text)
+    return result_columns
+
+
+def get_all_packets_by_regx(packets):
+    """
+    通过正则pcap获取所有包的数据
+    :param packets:
+    :return:
+    """
+    streams = b""
+    for pkt in packets:
+        if TCP in pkt:
+            streams += bytes(pkt[TCP].payload)
+    text = filter_visible_chars(streams)
+    pattern = r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d"
+    requests = re.split(f"(?={pattern})", text, re.M)
+    all_packets = []
+    for item in requests:
+        if len(re.findall(pattern, item)) != 0:
+            request_text = ""
+            response_text = ""
+            response_text_list = re.findall(r"HTTP/\d\.\d \d{3}[\s\S]*", item)
+            if len(response_text_list) != 0:
+                # 有响应数据
+                response_text = response_text_list[0]
+            if response_text == "":
+                # 没有响应数据，那么全是请求数据
+                request_text = item
+            else:
+                # 有响应数据，用正则获取请求数据
+                request_re = re.search(
+                    r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?\r\n\r\n", item)
+                if request_re:
+                    request_text = request_re.group(0)
+                else:
+                    request_text = ""
+            all_packets.append({"req": request_text, "res": response_text})
+    return all_packets
+
+
+def get_body(param, is_src):
+    body = param.split("\r\n\r\n")[1].strip()
+    return "" if body is None else body
+
+
+def get_header_value(header_set, value):
+    result = [item for item in header_set if value in item]
+    if len(result) != 0:
+        return result[0].replace(f"{value}:", "").strip()
+    else:
+        return ""
+
+
+def get_detail_by_package(packets_from_pcap, publicField, use_regx):
+    """
+    通过pcap的数量分离session并完善相关字段
+    :param packets_from_pcap: 通过PcAp解析出的包
+    :param publicField: 原始的session单条数据
+    :return: 完整的单条数据
+    """
+    res_field = publicField.copy()
+    if use_regx:
+        req = packets_from_pcap['req']
+        res = packets_from_pcap['res']
+    else:
+        res = packets_from_pcap["response"]
+        req = packets_from_pcap["request"]
+    res_field["initRTT"] = firstOrZero(res_field.get("initRTT", 0))
+    res_field["length"] = firstOrZero(res_field.get("length", 0))
+    request_lines = req.strip().split("\n")
+    http_request_lines = [item for item in request_lines if "HTTP" in item]
+    if len(http_request_lines) != 0:
+        first_line = http_request_lines[0].split(" ")
+        res_field['http.clientVersion'] = str(first_line[2]).replace("\n", "").replace("\r", "")
+        res_field['http.path'] = first_line[1]
+        res_field['http.method'] = first_line[0]
+    else:
+        res_field['http.clientVersion'] = ''
+        res_field['http.path'] = ''
+        res_field['http.method'] = ''
+    res_field['http.request-referer'] = get_header_value(header_set=request_lines, value="Referer")
+    res_field['http.request-content-type'] = get_header_value(header_set=request_lines,
+                                                              value="Content-Type")
+    res_field['http.hostTokens'] = get_header_value(header_set=request_lines, value="Host")
+
+    if use_regx:
+        res_field['plain_body_src'] = ""
+        res_field['plain_body_dst'] = ""
+        if content_type_is_plain(req):
+            res_field['plain_body_src'] = get_body(req, is_src=True)
+        if content_type_is_plain(res):
+            res_field['plain_body_dst'] = get_body(res, is_src=False)
+
+    response_lines = res.strip().split("\n")
+    http_response_lines = [item for item in response_lines if "HTTP" in item]
+    if len(http_response_lines) != 0:
+        first_line = http_response_lines[0].strip().split(" ")
+        res_field['http.statuscode'] = first_line[1]
+        res_field['http.serverVersion'] = first_line[0].split("/")[1]
+    else:
+        res_field['http.statuscode'] = ""
+        res_field['http.serverVersion'] = ""
+    res_field['http.response-server'] = get_header_value(header_set=response_lines, value="Server")
+    res_field['http.response-content-type'] = get_header_value(header_set=response_lines,
+                                                               value="Content-Type")
+    for response in list(set(response_lines + request_lines)):
+        key_value = response.replace("\r", "").split(":")
+        if len(key_value) == 2:
+            key = key_value[0].replace(" ", "").replace("-", "_").lower()
+            value = key_value[1].replace(" ", "")
+            if f"src_{key}" in src_dst_header:
+                res_field[f"src_{key}"] = value
+            if f"dst_{key}" in src_dst_header:
+                res_field[f"dst_{key}"] = value
+    return res_field

{xbase_util-0.4.2 → xbase_util-0.4.4}/xbase_util.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase-util
-Version: 0.4.2
+Version: 0.4.4
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt

{xbase_util-0.4.2 → xbase_util-0.4.4}/xbase_util.egg-info/SOURCES.txt

@@ -2,6 +2,7 @@ README.md
 setup.py
 xbase_util/__init__.py
 xbase_util/add_column_util.py
+xbase_util/dangerous_util.py
 xbase_util/es_db_util.py
 xbase_util/esreq.py
 xbase_util/geo_util.py

xbase_util-0.4.2/xbase_util/packet_util.py

@@ -1,93 +0,0 @@
-import re
-
-from scapy.layers.inet import TCP
-
-from xbase_util.xbase_constant import plain_content_type_columns, packetKeyname, src_dst_header, statisticHeader, \
-    features_key, plain_body_columns
-
-
-def content_type_is_plain(packet):
-    """
-    从单个包（包括header和body）中获取content-type并判断是否为可见类型
-    :param packet:
-    :return:
-    """
-    if ":" not in packet:
-        return False
-    for item in packet.replace("-", "_").replace(" ", "").lower().split("\n"):
-        if "content_type" in item:
-            if ":" not in item:
-                continue
-            content_type = item.split(":")[1].replace("\r", "").strip()
-            return content_type in plain_content_type_columns
-    return False
-
-
-def filter_visible_chars(data):
-    """
-    过滤不可见字符，仅保留可打印的ASCII字符
-    :param data:
-    :return:
-    """
-    return ''.join(chr(b) for b in data if 32 <= b <= 126 or b in (9, 10, 13))
-
-
-def get_all_columns(
-        contains_packet_column=False,
-        contains_src_dst_column=False,
-        contains_statistic_column=False,
-        contains_features_column=False,
-        contains_plain_body_column=False,
-        contains_pcap_flow_text=False
-):
-    result_columns = []
-    if contains_packet_column:
-        result_columns += packetKeyname
-    if contains_src_dst_column:
-        result_columns += src_dst_header
-    if contains_statistic_column:
-        result_columns += statisticHeader
-    if contains_features_column:
-        result_columns += features_key
-    if contains_plain_body_column:
-        result_columns += plain_body_columns
-    if contains_pcap_flow_text:
-        result_columns.append(contains_pcap_flow_text)
-    return result_columns
-
-
-def get_all_packets_by_regx(packets):
-    """
-    通过正则pcap获取所有包的数据
-    :param packets:
-    :return:
-    """
-    streams = b""
-    for pkt in packets:
-        if TCP in pkt:
-            streams += bytes(pkt[TCP].payload)
-    text = filter_visible_chars(streams)
-    pattern = r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d"
-    requests = re.split(f"(?={pattern})", text, re.M)
-    all_packets = []
-    for item in requests:
-        if len(re.findall(pattern, item)) != 0:
-            request_text = ""
-            response_text = ""
-            response_text_list = re.findall(r"HTTP/\d\.\d \d{3}[\s\S]*", item)
-            if len(response_text_list) != 0:
-                # 有响应数据
-                response_text = response_text_list[0]
-            if response_text == "":
-                # 没有响应数据，那么全是请求数据
-                request_text = item
-            else:
-                # 有响应数据，用正则获取请求数据
-                request_re = re.search(
-                    r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?\r\n\r\n", item)
-                if request_re:
-                    request_text = request_re.group(0)
-                else:
-                    request_text = ""
-            all_packets.append({"req": request_text, "res": response_text})
-    return all_packets