xbase-util 0.3.9__tar.gz → 0.4.1__tar.gz

{xbase_util-0.3.9 → xbase_util-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase_util
-Version: 0.3.9
+Version: 0.4.1
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt

{xbase_util-0.3.9 → xbase_util-0.4.1}/setup.py

@@ -3,7 +3,7 @@ from distutils.core import setup
 from setuptools import find_packages
 
 setup(name="xbase_util",
-      version="0.3.9",
+      version="0.4.1",
       description="网络安全基础工具",
       long_description="包含提取，预测，训练的基础工具",
       author="xyt",

{xbase_util-0.3.9 → xbase_util-0.4.1}/xbase_util/add_column_util.py

@@ -128,24 +128,40 @@ def parse_list(x):
     return x
 
 
-def handle_dns(origin_list, isDataFrame=False):
-    print("handle_dnslist")
+def handle_dns(origin_list, isDataFrame=False,use_tqdm=False):
     if not isDataFrame:
         origin_list = pd.DataFrame(origin_list)
-    origin_list["dnslist"] = origin_list['dns.host'].apply(parse_list)
-    origin_list['dns_host_is_long_domain'] = origin_list['dnslist'].apply(
-        lambda x: any(is_long_domain(domain) for domain in x))
-    origin_list['dns_host_is_random_characters'] = origin_list['dnslist'].apply(
-        lambda x: any(has_random_characters(domain) for domain in x))
-    origin_list['dns_host_is_special_characters'] = origin_list['dnslist'].apply(
-        lambda x: any(has_special_characters(domain) for domain in x))
-    origin_list['dns_host_is_large_subdomains'] = origin_list['dnslist'].apply(
-        lambda x: any(has_large_number_of_subdomains(domain) for domain in x))
-    origin_list['dns_host_is_danger_domain'] = origin_list['dnslist'].apply(
-        lambda x: any(is_danger_domain(domain) for domain in x))
-    origin_list['dns_host_is_danger_subdomain'] = origin_list['dnslist'].apply(
-        lambda x: any(is_danger_subdomain(domain) for domain in x))
-    origin_list['dns_host_is_uncommon_tld'] = origin_list['dnslist'].apply(
-        lambda x: any(has_uncommon_tld(domain) for domain in x))
+    if use_tqdm:
+        origin_list["dnslist"] = origin_list['dns.host'].progress_apply(parse_list)
+        origin_list['dns_host_is_long_domain'] = origin_list['dnslist'].progress_apply(
+            lambda x: any(is_long_domain(domain) for domain in x))
+        origin_list['dns_host_is_random_characters'] = origin_list['dnslist'].progress_apply(
+            lambda x: any(has_random_characters(domain) for domain in x))
+        origin_list['dns_host_is_special_characters'] = origin_list['dnslist'].progress_apply(
+            lambda x: any(has_special_characters(domain) for domain in x))
+        origin_list['dns_host_is_large_subdomains'] = origin_list['dnslist'].progress_apply(
+            lambda x: any(has_large_number_of_subdomains(domain) for domain in x))
+        origin_list['dns_host_is_danger_domain'] = origin_list['dnslist'].progress_apply(
+            lambda x: any(is_danger_domain(domain) for domain in x))
+        origin_list['dns_host_is_danger_subdomain'] = origin_list['dnslist'].progress_apply(
+            lambda x: any(is_danger_subdomain(domain) for domain in x))
+        origin_list['dns_host_is_uncommon_tld'] = origin_list['dnslist'].progress_apply(
+            lambda x: any(has_uncommon_tld(domain) for domain in x))
+    else:
+        origin_list["dnslist"] = origin_list['dns.host'].apply(parse_list)
+        origin_list['dns_host_is_long_domain'] = origin_list['dnslist'].apply(
+            lambda x: any(is_long_domain(domain) for domain in x))
+        origin_list['dns_host_is_random_characters'] = origin_list['dnslist'].apply(
+            lambda x: any(has_random_characters(domain) for domain in x))
+        origin_list['dns_host_is_special_characters'] = origin_list['dnslist'].apply(
+            lambda x: any(has_special_characters(domain) for domain in x))
+        origin_list['dns_host_is_large_subdomains'] = origin_list['dnslist'].apply(
+            lambda x: any(has_large_number_of_subdomains(domain) for domain in x))
+        origin_list['dns_host_is_danger_domain'] = origin_list['dnslist'].apply(
+            lambda x: any(is_danger_domain(domain) for domain in x))
+        origin_list['dns_host_is_danger_subdomain'] = origin_list['dnslist'].apply(
+            lambda x: any(is_danger_subdomain(domain) for domain in x))
+        origin_list['dns_host_is_uncommon_tld'] = origin_list['dnslist'].apply(
+            lambda x: any(has_uncommon_tld(domain) for domain in x))
     origin_list.drop(columns=['dnslist'], inplace=True)
     return origin_list

{xbase_util-0.3.9 → xbase_util-0.4.1}/xbase_util/handle_features_util.py

@@ -4,32 +4,11 @@ import traceback
 from urllib.parse import unquote
 
 import pandas as pd
-from tqdm import tqdm
-
-
-
-def handle_uri(data):
-    tqdm.pandas()
-    print(f"处理URI:{len(data)}")
-    # 定义正则表达式，确保精确匹配各种攻击特征
-    regex_patterns = {
-        "sql": re.compile(
-            r"\b(select|union|insert|update|delete|drop|--|#| or |' or '|information_schema|database\(\)|version\(\))\b",
-            re.IGNORECASE),
-        "xss": re.compile(r"(<script\b|javascript:|onload=|onclick=|<iframe\b|src=)", re.IGNORECASE),
-        "cmd": re.compile(
-            r"(/etc/passwd\b|/etc/shadow\b|;|&&|\||\$\(.+\)|\bcurl\b|\bwget\b|\bexec\b|\bsystem\b|cmd=|proc/self/environ)",
-            re.IGNORECASE),
-        "path": re.compile(r"(\.\./|\.\.%2f|\.\.%5c|\.\.\\|\.\.;|%2f%2e%2e%2f)", re.IGNORECASE),
-        "redirect": re.compile(r"(redirect=|url=|next=|redirect_uri=|redirect:|RedirectTo=)", re.IGNORECASE),
-        "danger": re.compile(
-            r"(%3C|%3E|%27|%22|%00|%2F|%5C|%3B|%7C|%28|%29|%20|%3D|%3A|%3F|%26|%23|%2B|%25|file://|<foo|xmlns:|/etc/passwd|windows/win\.ini)",
-            re.IGNORECASE),
-        "suspicious_ext": re.compile(
-            r"\.(exe|sh|py|pl|bak|php5|jspx|bat|cmd|pif|js|vbs|vbe|sct|ini|inf|tmp|swp|jar|java|class|ps1)\b",
-            re.IGNORECASE)
-    }
 
+from xbase_util.xbase_constant import regex_patterns
+
+
+def handle_uri(data, use_tqdm=True):
     # 定义多层解码函数，确保完全解码 URI
     def fully_decode_uri(uri):
         try:
@@ -55,7 +34,6 @@ def handle_uri(data):
             traceback.print_exc()
             exit(0)
 
-
         # 初始化统计变量
         param_count = 0
         path_depth = 0
@@ -97,14 +75,16 @@ def handle_uri(data):
             result[f"URI_FEATURES_EXTRA_contains_{key}"] = value
 
         return result
-    feature_data = data.progress_apply(process_row, axis=1, result_type="expand")
+
+    if use_tqdm:
+        feature_data = data.progress_apply(process_row, axis=1, result_type="expand")
+    else:
+        feature_data = data.apply(process_row, axis=1, result_type="expand")
     data = pd.concat([data, feature_data], axis=1)
     return data
 
 
-def handle_ua(data):
-    tqdm.pandas()
-    print("处理UA")
+def handle_ua(data, use_tqdm=True):
     data['http.useragent'] = data['http.useragent'].fillna('').astype(str)
     # 处理换行符及多余空格
     data['http.useragent'] = data['http.useragent'].str.replace(r'\s+', ' ', regex=True)
@@ -157,8 +137,14 @@ def handle_ua(data):
     data['UserAgent_language'] = data['http.useragent'].str.extract(r'\b([a-z]{2}-[A-Z]{2})\b', expand=False,
                                                                     flags=re.IGNORECASE).fillna("Unknown")
     # 统计 User-Agent 中的特殊字符个数
-    data['UserAgent_special_char_count'] = data['http.useragent'].progress_apply(
-        lambda x: len(re.findall(r'[!@#$%^&*\'=:|{}]', x, flags=re.IGNORECASE)))
+
+    if use_tqdm:
+        data['UserAgent_special_char_count'] = data['http.useragent'].progress_apply(
+            lambda x: len(re.findall(r'[!@#$%^&*\'=:|{}]', x, flags=re.IGNORECASE)))
+    else:
+        data['UserAgent_special_char_count'] = data['http.useragent'].apply(
+            lambda x: len(re.findall(r'[!@#$%^&*\'=:|{}]', x, flags=re.IGNORECASE)))
+
     # 更新 UserAgent_is_unknown 的计算逻辑
     data['UserAgent_is_unknown'] = data[['UserAgent_browser', 'UserAgent_os', 'UserAgent_platform']].isna().any(
         axis=1).fillna("Unknown")

{xbase_util-0.3.9 → xbase_util-0.4.1}/xbase_util/xbase_constant.py

@@ -1,4 +1,5 @@
 import os
+import re
 
 current_dir = os.path.dirname(__file__)
 parse_path = os.path.join(current_dir, '..', 'xbase_util_assets', 'arkimeparse.js')
@@ -203,4 +204,21 @@ features_key = [
     'URI_FEATURES_EXTRA_param_length_max', 'UserAgent_is_attack', 'UserAgent_is_enterprise', 'UserAgent_browser',
     'UserAgent_browser_version', 'UserAgent_os', 'UserAgent_os_version', 'UserAgent_device_type',
     'UserAgent_platform', 'UserAgent_is_bot', 'UserAgent_language', 'UserAgent_special_char_count',
-    'UserAgent_is_unknown']
+    'UserAgent_is_unknown']
+regex_patterns = {
+        "sql": re.compile(
+            r"\b(select|union|insert|update|delete|drop|--|#| or |' or '|information_schema|database\(\)|version\(\))\b",
+            re.IGNORECASE),
+        "xss": re.compile(r"(<script\b|javascript:|onload=|onclick=|<iframe\b|src=)", re.IGNORECASE),
+        "cmd": re.compile(
+            r"(/etc/passwd\b|/etc/shadow\b|;|&&|\||\$\(.+\)|\bcurl\b|\bwget\b|\bexec\b|\bsystem\b|cmd=|proc/self/environ)",
+            re.IGNORECASE),
+        "path": re.compile(r"(\.\./|\.\.%2f|\.\.%5c|\.\.\\|\.\.;|%2f%2e%2e%2f)", re.IGNORECASE),
+        "redirect": re.compile(r"(redirect=|url=|next=|redirect_uri=|redirect:|RedirectTo=)", re.IGNORECASE),
+        "danger": re.compile(
+            r"(%3C|%3E|%27|%22|%00|%2F|%5C|%3B|%7C|%28|%29|%20|%3D|%3A|%3F|%26|%23|%2B|%25|file://|<foo|xmlns:|/etc/passwd|windows/win\.ini)",
+            re.IGNORECASE),
+        "suspicious_ext": re.compile(
+            r"\.(exe|sh|py|pl|bak|php5|jspx|bat|cmd|pif|js|vbs|vbe|sct|ini|inf|tmp|swp|jar|java|class|ps1)\b",
+            re.IGNORECASE)
+    }

{xbase_util-0.3.9 → xbase_util-0.4.1}/xbase_util.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase-util
-Version: 0.3.9
+Version: 0.4.1
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt