xbase-util 0.4.1__tar.gz → 0.4.3__tar.gz

{xbase_util-0.4.1 → xbase_util-0.4.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase_util
-Version: 0.4.1
+Version: 0.4.3
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt

{xbase_util-0.4.1 → xbase_util-0.4.3}/setup.py

@@ -3,7 +3,7 @@ from distutils.core import setup
 from setuptools import find_packages
 
 setup(name="xbase_util",
-      version="0.4.1",
+      version="0.4.3",
       description="网络安全基础工具",
       long_description="包含提取，预测，训练的基础工具",
       author="xyt",

xbase_util-0.4.3/xbase_util/packet_util.py

@@ -0,0 +1,171 @@
+import re
+
+from scapy.layers.inet import TCP
+
+from xbase_util.xbase_constant import plain_content_type_columns, packetKeyname, src_dst_header, statisticHeader, \
+    features_key, plain_body_columns
+from xbase_util.xbase_util import firstOrZero
+
+
+def content_type_is_plain(packet):
+    """
+    从单个包（包括header和body）中获取content-type并判断是否为可见类型
+    :param packet:
+    :return:
+    """
+    if ":" not in packet:
+        return False
+    for item in packet.replace("-", "_").replace(" ", "").lower().split("\n"):
+        if "content_type" in item:
+            if ":" not in item:
+                continue
+            content_type = item.split(":")[1].replace("\r", "").strip()
+            return content_type in plain_content_type_columns
+    return False
+
+
+def filter_visible_chars(data):
+    """
+    过滤不可见字符，仅保留可打印的ASCII字符
+    :param data:
+    :return:
+    """
+    return ''.join(chr(b) for b in data if 32 <= b <= 126 or b in (9, 10, 13))
+
+
+def get_all_columns(
+        contains_packet_column=False,
+        contains_src_dst_column=False,
+        contains_statistic_column=False,
+        contains_features_column=False,
+        contains_plain_body_column=False,
+        contains_pcap_flow_text=False
+):
+    result_columns = []
+    if contains_packet_column:
+        result_columns += packetKeyname
+    if contains_src_dst_column:
+        result_columns += src_dst_header
+    if contains_statistic_column:
+        result_columns += statisticHeader
+    if contains_features_column:
+        result_columns += features_key
+    if contains_plain_body_column:
+        result_columns += plain_body_columns
+    if contains_pcap_flow_text:
+        result_columns.append(contains_pcap_flow_text)
+    return result_columns
+
+
+def get_all_packets_by_regx(packets):
+    """
+    通过正则pcap获取所有包的数据
+    :param packets:
+    :return:
+    """
+    streams = b""
+    for pkt in packets:
+        if TCP in pkt:
+            streams += bytes(pkt[TCP].payload)
+    text = filter_visible_chars(streams)
+    pattern = r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d"
+    requests = re.split(f"(?={pattern})", text, re.M)
+    all_packets = []
+    for item in requests:
+        if len(re.findall(pattern, item)) != 0:
+            request_text = ""
+            response_text = ""
+            response_text_list = re.findall(r"HTTP/\d\.\d \d{3}[\s\S]*", item)
+            if len(response_text_list) != 0:
+                # 有响应数据
+                response_text = response_text_list[0]
+            if response_text == "":
+                # 没有响应数据，那么全是请求数据
+                request_text = item
+            else:
+                # 有响应数据，用正则获取请求数据
+                request_re = re.search(
+                    r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?\r\n\r\n", item)
+                if request_re:
+                    request_text = request_re.group(0)
+                else:
+                    request_text = ""
+            all_packets.append({"req": request_text, "res": response_text})
+    return all_packets
+
+
+def get_body(param, is_src):
+    body = param.split("\r\n\r\n")[1].strip()
+    return "" if body is None else body
+
+
+def get_header_value(header_set, value):
+    result = [item for item in header_set if value in item]
+    if len(result) != 0:
+        return result[0].replace(f"{value}:", "").strip()
+    else:
+        return ""
+
+
+def get_detail_by_package(packets_from_pcap, publicField, use_regx):
+    """
+    通过pcap的数量分离session并完善相关字段
+    :param packets_from_pcap: 通过PcAp解析出的包
+    :param publicField: 原始的session单条数据
+    :return: 完整的单条数据
+    """
+    res_field = publicField.copy()
+    if use_regx:
+        req = packets_from_pcap['req']
+        res = packets_from_pcap['res']
+    else:
+        res = packets_from_pcap["response"]
+        req = packets_from_pcap["request"]
+    res_field["initRTT"] = firstOrZero(res_field.get("initRTT", 0))
+    res_field["length"] = firstOrZero(res_field.get("length", 0))
+    request_lines = req.strip().split("\n")
+    http_request_lines = [item for item in request_lines if "HTTP" in item]
+    if len(http_request_lines) != 0:
+        first_line = http_request_lines[0].split(" ")
+        res_field['http.clientVersion'] = str(first_line[2]).replace("\n", "").replace("\r", "")
+        res_field['http.path'] = first_line[1]
+        res_field['http.method'] = first_line[0]
+    else:
+        res_field['http.clientVersion'] = ''
+        res_field['http.path'] = ''
+        res_field['http.method'] = ''
+    res_field['http.request-referer'] = get_header_value(header_set=request_lines, value="Referer")
+    res_field['http.request-content-type'] = get_header_value(header_set=request_lines,
+                                                              value="Content-Type")
+    res_field['http.hostTokens'] = get_header_value(header_set=request_lines, value="Host")
+
+    if use_regx:
+        res_field['plain_body_src'] = ""
+        res_field['plain_body_dst'] = ""
+        if content_type_is_plain(req):
+            res_field['plain_body_src'] = get_body(req, is_src=True)
+        if content_type_is_plain(res):
+            res_field['plain_body_dst'] = get_body(res, is_src=False)
+
+    response_lines = res.strip().split("\n")
+    http_response_lines = [item for item in response_lines if "HTTP" in item]
+    if len(http_response_lines) != 0:
+        first_line = http_response_lines[0].strip().split(" ")
+        res_field['http.statuscode'] = first_line[1]
+        res_field['http.serverVersion'] = first_line[0].split("/")[1]
+    else:
+        res_field['http.statuscode'] = ""
+        res_field['http.serverVersion'] = ""
+    res_field['http.response-server'] = get_header_value(header_set=response_lines, value="Server")
+    res_field['http.response-content-type'] = get_header_value(header_set=response_lines,
+                                                               value="Content-Type")
+    for response in list(set(response_lines + request_lines)):
+        key_value = response.replace("\r", "").split(":")
+        if len(key_value) == 2:
+            key = key_value[0].replace(" ", "").replace("-", "_").lower()
+            value = key_value[1].replace(" ", "")
+            if f"src_{key}" in src_dst_header:
+                res_field[f"src_{key}"] = value
+            if f"dst_{key}" in src_dst_header:
+                res_field[f"dst_{key}"] = value
+    return res_field

{xbase_util-0.4.1 → xbase_util-0.4.3}/xbase_util/pcap_util.py

@@ -4,7 +4,6 @@ import struct
 import time
 import zlib
 from datetime import datetime
-
 from Crypto.Cipher import AES
 from zstandard import ZstdDecompressor
 

{xbase_util-0.4.1 → xbase_util-0.4.3}/xbase_util/xbase_constant.py

@@ -206,19 +206,82 @@ features_key = [
     'UserAgent_platform', 'UserAgent_is_bot', 'UserAgent_language', 'UserAgent_special_char_count',
     'UserAgent_is_unknown']
 regex_patterns = {
-        "sql": re.compile(
-            r"\b(select|union|insert|update|delete|drop|--|#| or |' or '|information_schema|database\(\)|version\(\))\b",
-            re.IGNORECASE),
-        "xss": re.compile(r"(<script\b|javascript:|onload=|onclick=|<iframe\b|src=)", re.IGNORECASE),
-        "cmd": re.compile(
-            r"(/etc/passwd\b|/etc/shadow\b|;|&&|\||\$\(.+\)|\bcurl\b|\bwget\b|\bexec\b|\bsystem\b|cmd=|proc/self/environ)",
-            re.IGNORECASE),
-        "path": re.compile(r"(\.\./|\.\.%2f|\.\.%5c|\.\.\\|\.\.;|%2f%2e%2e%2f)", re.IGNORECASE),
-        "redirect": re.compile(r"(redirect=|url=|next=|redirect_uri=|redirect:|RedirectTo=)", re.IGNORECASE),
-        "danger": re.compile(
-            r"(%3C|%3E|%27|%22|%00|%2F|%5C|%3B|%7C|%28|%29|%20|%3D|%3A|%3F|%26|%23|%2B|%25|file://|<foo|xmlns:|/etc/passwd|windows/win\.ini)",
-            re.IGNORECASE),
-        "suspicious_ext": re.compile(
-            r"\.(exe|sh|py|pl|bak|php5|jspx|bat|cmd|pif|js|vbs|vbe|sct|ini|inf|tmp|swp|jar|java|class|ps1)\b",
-            re.IGNORECASE)
-    }
+    "sql": re.compile(
+        r"\b(select|union|insert|update|delete|drop|--|#| or |' or '|information_schema|database\(\)|version\(\))\b",
+        re.IGNORECASE),
+    "xss": re.compile(r"(<script\b|javascript:|onload=|onclick=|<iframe\b|src=)", re.IGNORECASE),
+    "cmd": re.compile(
+        r"(/etc/passwd\b|/etc/shadow\b|;|&&|\||\$\(.+\)|\bcurl\b|\bwget\b|\bexec\b|\bsystem\b|cmd=|proc/self/environ)",
+        re.IGNORECASE),
+    "path": re.compile(r"(\.\./|\.\.%2f|\.\.%5c|\.\.\\|\.\.;|%2f%2e%2e%2f)", re.IGNORECASE),
+    "redirect": re.compile(r"(redirect=|url=|next=|redirect_uri=|redirect:|RedirectTo=)", re.IGNORECASE),
+    "danger": re.compile(
+        r"(%3C|%3E|%27|%22|%00|%2F|%5C|%3B|%7C|%28|%29|%20|%3D|%3A|%3F|%26|%23|%2B|%25|file://|<foo|xmlns:|/etc/passwd|windows/win\.ini)",
+        re.IGNORECASE),
+    "suspicious_ext": re.compile(
+        r"\.(exe|sh|py|pl|bak|php5|jspx|bat|cmd|pif|js|vbs|vbe|sct|ini|inf|tmp|swp|jar|java|class|ps1)\b",
+        re.IGNORECASE)
+}
+# 可见的content-type值
+plain_content_type_columns = ['text/json;charset=gbk',
+                              'application/xml;charset=gbk', 'application/xml;charset=utf_8', 'application/tlt_notify',
+                              'application/json;charset=gbk', 'text/xml;charset=utf_8', 'application/json',
+                              'text/csv;charset=utf_8', 'application/json;charse=utf_8',
+                              'application/soap+xml;charset=utf_8;action="urn:dopricetaxseparated"',
+                              'text/xml;charset=gbk', 'text/xml', 'application/x_cm_json;charset=utf_8',
+                              'application/xml;tz=utc', 'text/xml;charset="utf_8"', 'application/x_java_archive',
+                              'application/msword', 'image/png', 'application/xml',
+                              'application/x_stapler_method_invocation;charset=utf_8', 'text/plain;charset=iso_8859_1',
+                              'application/x_www_form_urlencoded;charset=utf_8', 'text/plain;charset=gbk',
+                              'application/octet_stream;charset=utf_8', 'application/x_tika_ooxml',
+                              'application/soap+xml;charset=utf_8;action="urn:sendcommand"', 'application/dns_message',
+                              'application/json;charset=utf_8', 'application/vnd.docker.distribution.manifest.v2+json',
+                              'application/vnd.elasticsearch+json;compatible_with=8', 'off/ping', 'text/plain',
+                              'application/x_git_upload_pack_request', 'application/json;charset=gbk',
+                              'text/html;charset=iso_8859_1', 'text/http;charset=utf_8',
+                              'application/soap+xml;charset=gbk', 'text/html',
+                              'application/vnd.openxmlformats_officedocument.spreadsheetml.sheet',
+                              'application/x_www_form_urlencoded;charset=gbk', 'text/plain;charset=utf_8',
+                              'text/html;charset=gbk', 'application/soap+xml;charset=gbk;',
+                              'application/x_www_form_urlencoded', 'application/x_ndjson', 'text/xml;charset=gbk',
+                              'application/json;chartset=utf_8',
+                              'application/soap+xml;charset=utf_8;action="urn:getcostbyruleengine"',
+                              'application/json_rpc', 'text/json;charset=utf_8', 'application/json;charset=utf8',
+                              'application/xml;charset=utf_8', 'application/x_www_form_urlencoded;charset=gbk',
+                              'application/soap+xml;charset=utf_8;', 'application/merge_patch+json',
+                              'application/json;', 'text/xml;charset="utf_16le"', 'text/html;charset=utf_8']
+packetKeyname = ['id', 'segmentCnt', 'tcpflags.rst', 'tcpflags.ack', 'tcpflags.syn', 'tcpflags.urg', 'tcpflags.psh',
+                 'tcpflags.syn-ack', 'tcpflags.fin', 'source.ip', 'destination.ip', 'source.port', 'source.packets',
+                 'source.bytes', 'destination.port', 'destination.bytes', 'destination.packets', 'initRTT',
+                 'firstPacket', 'lastPacket', 'ipProtocol', 'protocolCnt', 'protocol', 'server.bytes', 'totDataBytes',
+                 'network.packets', 'network.bytes', 'length', 'client.bytes', 'http.uri',
+                 'http.response-content-type', 'http.bodyMagicCnt', 'http.statuscodeCnt', 'http.clientVersionCnt',
+                 'http.response-content-typeCnt', 'http.xffIpCnt', 'http.requestHeaderCnt', 'http.serverVersion',
+                 'http.responseHeaderCnt', 'http.xffIp', 'http.clientVersion', 'http.uriTokens',
+                 'http.request-refererCnt', 'http.useragentCnt', 'http.statuscode', 'http.bodyMagic', 'http.methodCnt',
+                 'http.request-content-type', 'http.uriCnt', 'http.serverVersionCnt', 'http.useragent', 'http.keyCnt',
+                 'http.request-referer', 'http.path', 'http.hostCnt', 'http.response-server', 'http.pathCnt',
+                 'http.useragentTokens', 'http.method-GET', 'http.method', 'http.key', 'http.hostTokens',
+                 'http.requestHeader', 'http.responseHeader', 'http.method-POST', 'dns.ASN', 'dns.RIR', 'dns.GEO',
+                 'dns.alpn', 'dns.alpnCnt', 'dns.ip', 'dns.host', 'dns.ipCnt', 'dns.OpCode', 'dns.OpCodeCnt',
+                 'dns.Puny', 'dns.PunyCnt', 'dns.QueryClass', 'dns.QueryClassCnt', 'dns.QueryType', 'dns.QueryTypeCnt',
+                 'dns.status', 'dns.statusCnt', 'tls.cipher', 'tls.cipherCnt', 'tls.dstSessionId', 'tls.ja3',
+                 'tls.ja3Cnt', 'tls.ja3s', 'tls.ja3sCnt', 'tls.ja4', 'tls.ja4Cnt', 'tls.srcSessionId', 'tls.version',
+                 'tls.versionCnt', 'tls.ja4_r', 'tls.ja4_rCnt', 'packetPos', 'source.ip_Country_IsoCode',
+                 'source.ip_Country_Name', 'source.ip_Country_SpecificName',
+                 'source.ip_Country_SpecificIsoCode', 'source.ip_City_Name', 'source.ip_City_PostalCode',
+                 'source.ip_Location_Latitude', 'source.ip_Location_Longitude', 'destination.ip_Country_IsoCode',
+                 'destination.ip_Country_Name', 'destination.ip_Country_SpecificName',
+                 'destination.ip_Country_SpecificIsoCode', 'destination.ip_City_Name',
+                 'destination.ip_City_PostalCode', 'destination.ip_Location_Latitude',
+                 'destination.ip_Location_Longitude', 'http.uri_length_mean', 'http.uri_length_var',
+                 "http.uri_param_count_mean", "http.uri_param_count_var", "http.uri_depth_mean", "http.uri_depth_var",
+                 "http.uri_filename_length_mean", "http.uri_filename_length_var", "dns_domain_length_mean",
+                 "dns_domain_length_var", "traffic_type", "PROTOCOL", "DENY_METHOD", "THREAT_SUMMARY", "SEVERITY",
+                 "dns_domain_length", "dns_domain_suffix", "dns_domain", "dns_domain_suffix_length", "dns_base_domain",
+                 "dns_base_domain_length", "req_res_period_mean", "req_res_period_var", "status_code_1x_count",
+                 "status_code_2x_count", "status_code_3x_count", "status_code_4x_count", "status_code_5x_count",
+                 "req_bytes_percentage", "res_bytes_percentage", "cookie_end_with_semicolon_count",
+                 "ua_duplicate_count"]
+plain_body_columns = ["plain_body_src",
+                      "plain_body_dst"]

{xbase_util-0.4.1 → xbase_util-0.4.3}/xbase_util.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase-util
-Version: 0.4.1
+Version: 0.4.3
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt

{xbase_util-0.4.1 → xbase_util-0.4.3}/xbase_util.egg-info/SOURCES.txt

@@ -6,6 +6,7 @@ xbase_util/es_db_util.py
 xbase_util/esreq.py
 xbase_util/geo_util.py
 xbase_util/handle_features_util.py
+xbase_util/packet_util.py
 xbase_util/pcap_util.py
 xbase_util/xbase_constant.py
 xbase_util/xbase_util.py