xbase-util 0.4.0__tar.gz → 0.4.2__tar.gz

{xbase_util-0.4.0 → xbase_util-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase_util
-Version: 0.4.0
+Version: 0.4.2
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt

{xbase_util-0.4.0 → xbase_util-0.4.2}/setup.py

@@ -3,7 +3,7 @@ from distutils.core import setup
 from setuptools import find_packages
 
 setup(name="xbase_util",
-      version="0.4.0",
+      version="0.4.2",
       description="网络安全基础工具",
       long_description="包含提取，预测，训练的基础工具",
       author="xyt",

{xbase_util-0.4.0 → xbase_util-0.4.2}/xbase_util/handle_features_util.py

@@ -4,30 +4,11 @@ import traceback
 from urllib.parse import unquote
 
 import pandas as pd
-from tqdm import tqdm
-
-
-
-def handle_uri(data,use_tqdm=True):
-    # 定义正则表达式，确保精确匹配各种攻击特征
-    regex_patterns = {
-        "sql": re.compile(
-            r"\b(select|union|insert|update|delete|drop|--|#| or |' or '|information_schema|database\(\)|version\(\))\b",
-            re.IGNORECASE),
-        "xss": re.compile(r"(<script\b|javascript:|onload=|onclick=|<iframe\b|src=)", re.IGNORECASE),
-        "cmd": re.compile(
-            r"(/etc/passwd\b|/etc/shadow\b|;|&&|\||\$\(.+\)|\bcurl\b|\bwget\b|\bexec\b|\bsystem\b|cmd=|proc/self/environ)",
-            re.IGNORECASE),
-        "path": re.compile(r"(\.\./|\.\.%2f|\.\.%5c|\.\.\\|\.\.;|%2f%2e%2e%2f)", re.IGNORECASE),
-        "redirect": re.compile(r"(redirect=|url=|next=|redirect_uri=|redirect:|RedirectTo=)", re.IGNORECASE),
-        "danger": re.compile(
-            r"(%3C|%3E|%27|%22|%00|%2F|%5C|%3B|%7C|%28|%29|%20|%3D|%3A|%3F|%26|%23|%2B|%25|file://|<foo|xmlns:|/etc/passwd|windows/win\.ini)",
-            re.IGNORECASE),
-        "suspicious_ext": re.compile(
-            r"\.(exe|sh|py|pl|bak|php5|jspx|bat|cmd|pif|js|vbs|vbe|sct|ini|inf|tmp|swp|jar|java|class|ps1)\b",
-            re.IGNORECASE)
-    }
 
+from xbase_util.xbase_constant import regex_patterns
+
+
+def handle_uri(data, use_tqdm=True):
     # 定义多层解码函数，确保完全解码 URI
     def fully_decode_uri(uri):
         try:
@@ -53,7 +34,6 @@ def handle_uri(data,use_tqdm=True):
             traceback.print_exc()
             exit(0)
 
-
         # 初始化统计变量
         param_count = 0
         path_depth = 0
@@ -95,6 +75,7 @@ def handle_uri(data,use_tqdm=True):
             result[f"URI_FEATURES_EXTRA_contains_{key}"] = value
 
         return result
+
     if use_tqdm:
         feature_data = data.progress_apply(process_row, axis=1, result_type="expand")
     else:
@@ -103,7 +84,7 @@ def handle_uri(data,use_tqdm=True):
     return data
 
 
-def handle_ua(data,use_tqdm=True):
+def handle_ua(data, use_tqdm=True):
     data['http.useragent'] = data['http.useragent'].fillna('').astype(str)
     # 处理换行符及多余空格
     data['http.useragent'] = data['http.useragent'].str.replace(r'\s+', ' ', regex=True)

xbase_util-0.4.2/xbase_util/packet_util.py

@@ -0,0 +1,93 @@
+import re
+
+from scapy.layers.inet import TCP
+
+from xbase_util.xbase_constant import plain_content_type_columns, packetKeyname, src_dst_header, statisticHeader, \
+    features_key, plain_body_columns
+
+
+def content_type_is_plain(packet):
+    """
+    从单个包（包括header和body）中获取content-type并判断是否为可见类型
+    :param packet:
+    :return:
+    """
+    if ":" not in packet:
+        return False
+    for item in packet.replace("-", "_").replace(" ", "").lower().split("\n"):
+        if "content_type" in item:
+            if ":" not in item:
+                continue
+            content_type = item.split(":")[1].replace("\r", "").strip()
+            return content_type in plain_content_type_columns
+    return False
+
+
+def filter_visible_chars(data):
+    """
+    过滤不可见字符，仅保留可打印的ASCII字符
+    :param data:
+    :return:
+    """
+    return ''.join(chr(b) for b in data if 32 <= b <= 126 or b in (9, 10, 13))
+
+
+def get_all_columns(
+        contains_packet_column=False,
+        contains_src_dst_column=False,
+        contains_statistic_column=False,
+        contains_features_column=False,
+        contains_plain_body_column=False,
+        contains_pcap_flow_text=False
+):
+    result_columns = []
+    if contains_packet_column:
+        result_columns += packetKeyname
+    if contains_src_dst_column:
+        result_columns += src_dst_header
+    if contains_statistic_column:
+        result_columns += statisticHeader
+    if contains_features_column:
+        result_columns += features_key
+    if contains_plain_body_column:
+        result_columns += plain_body_columns
+    if contains_pcap_flow_text:
+        result_columns.append(contains_pcap_flow_text)
+    return result_columns
+
+
+def get_all_packets_by_regx(packets):
+    """
+    通过正则pcap获取所有包的数据
+    :param packets:
+    :return:
+    """
+    streams = b""
+    for pkt in packets:
+        if TCP in pkt:
+            streams += bytes(pkt[TCP].payload)
+    text = filter_visible_chars(streams)
+    pattern = r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d"
+    requests = re.split(f"(?={pattern})", text, re.M)
+    all_packets = []
+    for item in requests:
+        if len(re.findall(pattern, item)) != 0:
+            request_text = ""
+            response_text = ""
+            response_text_list = re.findall(r"HTTP/\d\.\d \d{3}[\s\S]*", item)
+            if len(response_text_list) != 0:
+                # 有响应数据
+                response_text = response_text_list[0]
+            if response_text == "":
+                # 没有响应数据，那么全是请求数据
+                request_text = item
+            else:
+                # 有响应数据，用正则获取请求数据
+                request_re = re.search(
+                    r"(GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH) \/[^\s]* HTTP\/\d\.\d[\s\S]*?\r\n\r\n", item)
+                if request_re:
+                    request_text = request_re.group(0)
+                else:
+                    request_text = ""
+            all_packets.append({"req": request_text, "res": response_text})
+    return all_packets

{xbase_util-0.4.0 → xbase_util-0.4.2}/xbase_util/pcap_util.py

@@ -4,7 +4,6 @@ import struct
 import time
 import zlib
 from datetime import datetime
-
 from Crypto.Cipher import AES
 from zstandard import ZstdDecompressor
 

{xbase_util-0.4.0 → xbase_util-0.4.2}/xbase_util/xbase_constant.py

@@ -1,4 +1,5 @@
 import os
+import re
 
 current_dir = os.path.dirname(__file__)
 parse_path = os.path.join(current_dir, '..', 'xbase_util_assets', 'arkimeparse.js')
@@ -203,4 +204,84 @@ features_key = [
     'URI_FEATURES_EXTRA_param_length_max', 'UserAgent_is_attack', 'UserAgent_is_enterprise', 'UserAgent_browser',
     'UserAgent_browser_version', 'UserAgent_os', 'UserAgent_os_version', 'UserAgent_device_type',
     'UserAgent_platform', 'UserAgent_is_bot', 'UserAgent_language', 'UserAgent_special_char_count',
-    'UserAgent_is_unknown']
+    'UserAgent_is_unknown']
+regex_patterns = {
+    "sql": re.compile(
+        r"\b(select|union|insert|update|delete|drop|--|#| or |' or '|information_schema|database\(\)|version\(\))\b",
+        re.IGNORECASE),
+    "xss": re.compile(r"(<script\b|javascript:|onload=|onclick=|<iframe\b|src=)", re.IGNORECASE),
+    "cmd": re.compile(
+        r"(/etc/passwd\b|/etc/shadow\b|;|&&|\||\$\(.+\)|\bcurl\b|\bwget\b|\bexec\b|\bsystem\b|cmd=|proc/self/environ)",
+        re.IGNORECASE),
+    "path": re.compile(r"(\.\./|\.\.%2f|\.\.%5c|\.\.\\|\.\.;|%2f%2e%2e%2f)", re.IGNORECASE),
+    "redirect": re.compile(r"(redirect=|url=|next=|redirect_uri=|redirect:|RedirectTo=)", re.IGNORECASE),
+    "danger": re.compile(
+        r"(%3C|%3E|%27|%22|%00|%2F|%5C|%3B|%7C|%28|%29|%20|%3D|%3A|%3F|%26|%23|%2B|%25|file://|<foo|xmlns:|/etc/passwd|windows/win\.ini)",
+        re.IGNORECASE),
+    "suspicious_ext": re.compile(
+        r"\.(exe|sh|py|pl|bak|php5|jspx|bat|cmd|pif|js|vbs|vbe|sct|ini|inf|tmp|swp|jar|java|class|ps1)\b",
+        re.IGNORECASE)
+}
+# 可见的content-type值
+plain_content_type_columns = ['text/json;charset=gbk',
+                              'application/xml;charset=gbk', 'application/xml;charset=utf_8', 'application/tlt_notify',
+                              'application/json;charset=gbk', 'text/xml;charset=utf_8', 'application/json',
+                              'text/csv;charset=utf_8', 'application/json;charse=utf_8',
+                              'application/soap+xml;charset=utf_8;action="urn:dopricetaxseparated"',
+                              'text/xml;charset=gbk', 'text/xml', 'application/x_cm_json;charset=utf_8',
+                              'application/xml;tz=utc', 'text/xml;charset="utf_8"', 'application/x_java_archive',
+                              'application/msword', 'image/png', 'application/xml',
+                              'application/x_stapler_method_invocation;charset=utf_8', 'text/plain;charset=iso_8859_1',
+                              'application/x_www_form_urlencoded;charset=utf_8', 'text/plain;charset=gbk',
+                              'application/octet_stream;charset=utf_8', 'application/x_tika_ooxml',
+                              'application/soap+xml;charset=utf_8;action="urn:sendcommand"', 'application/dns_message',
+                              'application/json;charset=utf_8', 'application/vnd.docker.distribution.manifest.v2+json',
+                              'application/vnd.elasticsearch+json;compatible_with=8', 'off/ping', 'text/plain',
+                              'application/x_git_upload_pack_request', 'application/json;charset=gbk',
+                              'text/html;charset=iso_8859_1', 'text/http;charset=utf_8',
+                              'application/soap+xml;charset=gbk', 'text/html',
+                              'application/vnd.openxmlformats_officedocument.spreadsheetml.sheet',
+                              'application/x_www_form_urlencoded;charset=gbk', 'text/plain;charset=utf_8',
+                              'text/html;charset=gbk', 'application/soap+xml;charset=gbk;',
+                              'application/x_www_form_urlencoded', 'application/x_ndjson', 'text/xml;charset=gbk',
+                              'application/json;chartset=utf_8',
+                              'application/soap+xml;charset=utf_8;action="urn:getcostbyruleengine"',
+                              'application/json_rpc', 'text/json;charset=utf_8', 'application/json;charset=utf8',
+                              'application/xml;charset=utf_8', 'application/x_www_form_urlencoded;charset=gbk',
+                              'application/soap+xml;charset=utf_8;', 'application/merge_patch+json',
+                              'application/json;', 'text/xml;charset="utf_16le"', 'text/html;charset=utf_8']
+packetKeyname = ['id', 'segmentCnt', 'tcpflags.rst', 'tcpflags.ack', 'tcpflags.syn', 'tcpflags.urg', 'tcpflags.psh',
+                 'tcpflags.syn-ack', 'tcpflags.fin', 'source.ip', 'destination.ip', 'source.port', 'source.packets',
+                 'source.bytes', 'destination.port', 'destination.bytes', 'destination.packets', 'initRTT',
+                 'firstPacket', 'lastPacket', 'ipProtocol', 'protocolCnt', 'protocol', 'server.bytes', 'totDataBytes',
+                 'network.packets', 'network.bytes', 'length', 'client.bytes', 'http.uri',
+                 'http.response-content-type', 'http.bodyMagicCnt', 'http.statuscodeCnt', 'http.clientVersionCnt',
+                 'http.response-content-typeCnt', 'http.xffIpCnt', 'http.requestHeaderCnt', 'http.serverVersion',
+                 'http.responseHeaderCnt', 'http.xffIp', 'http.clientVersion', 'http.uriTokens',
+                 'http.request-refererCnt', 'http.useragentCnt', 'http.statuscode', 'http.bodyMagic', 'http.methodCnt',
+                 'http.request-content-type', 'http.uriCnt', 'http.serverVersionCnt', 'http.useragent', 'http.keyCnt',
+                 'http.request-referer', 'http.path', 'http.hostCnt', 'http.response-server', 'http.pathCnt',
+                 'http.useragentTokens', 'http.method-GET', 'http.method', 'http.key', 'http.hostTokens',
+                 'http.requestHeader', 'http.responseHeader', 'http.method-POST', 'dns.ASN', 'dns.RIR', 'dns.GEO',
+                 'dns.alpn', 'dns.alpnCnt', 'dns.ip', 'dns.host', 'dns.ipCnt', 'dns.OpCode', 'dns.OpCodeCnt',
+                 'dns.Puny', 'dns.PunyCnt', 'dns.QueryClass', 'dns.QueryClassCnt', 'dns.QueryType', 'dns.QueryTypeCnt',
+                 'dns.status', 'dns.statusCnt', 'tls.cipher', 'tls.cipherCnt', 'tls.dstSessionId', 'tls.ja3',
+                 'tls.ja3Cnt', 'tls.ja3s', 'tls.ja3sCnt', 'tls.ja4', 'tls.ja4Cnt', 'tls.srcSessionId', 'tls.version',
+                 'tls.versionCnt', 'tls.ja4_r', 'tls.ja4_rCnt', 'packetPos', 'source.ip_Country_IsoCode',
+                 'source.ip_Country_Name', 'source.ip_Country_SpecificName',
+                 'source.ip_Country_SpecificIsoCode', 'source.ip_City_Name', 'source.ip_City_PostalCode',
+                 'source.ip_Location_Latitude', 'source.ip_Location_Longitude', 'destination.ip_Country_IsoCode',
+                 'destination.ip_Country_Name', 'destination.ip_Country_SpecificName',
+                 'destination.ip_Country_SpecificIsoCode', 'destination.ip_City_Name',
+                 'destination.ip_City_PostalCode', 'destination.ip_Location_Latitude',
+                 'destination.ip_Location_Longitude', 'http.uri_length_mean', 'http.uri_length_var',
+                 "http.uri_param_count_mean", "http.uri_param_count_var", "http.uri_depth_mean", "http.uri_depth_var",
+                 "http.uri_filename_length_mean", "http.uri_filename_length_var", "dns_domain_length_mean",
+                 "dns_domain_length_var", "traffic_type", "PROTOCOL", "DENY_METHOD", "THREAT_SUMMARY", "SEVERITY",
+                 "dns_domain_length", "dns_domain_suffix", "dns_domain", "dns_domain_suffix_length", "dns_base_domain",
+                 "dns_base_domain_length", "req_res_period_mean", "req_res_period_var", "status_code_1x_count",
+                 "status_code_2x_count", "status_code_3x_count", "status_code_4x_count", "status_code_5x_count",
+                 "req_bytes_percentage", "res_bytes_percentage", "cookie_end_with_semicolon_count",
+                 "ua_duplicate_count"]
+plain_body_columns = ["plain_body_src",
+                      "plain_body_dst"]

{xbase_util-0.4.0 → xbase_util-0.4.2}/xbase_util.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase-util
-Version: 0.4.0
+Version: 0.4.2
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt

{xbase_util-0.4.0 → xbase_util-0.4.2}/xbase_util.egg-info/SOURCES.txt

@@ -6,6 +6,7 @@ xbase_util/es_db_util.py
 xbase_util/esreq.py
 xbase_util/geo_util.py
 xbase_util/handle_features_util.py
+xbase_util/packet_util.py
 xbase_util/pcap_util.py
 xbase_util/xbase_constant.py
 xbase_util/xbase_util.py