xbase-util 0.0.8__tar.gz → 0.1.0__tar.gz

{xbase_util-0.0.8 → xbase_util-0.1.0}/PKG-INFO

@@ -1,7 +1,8 @@
 Metadata-Version: 2.1
 Name: xbase_util
-Version: 0.0.8
+Version: 0.1.0
 Summary: 网络安全基础工具
+Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt
 Author-email: 2506564278@qq.com
 License: <MIT License>

{xbase_util-0.0.8 → xbase_util-0.1.0}/setup.py

@@ -3,13 +3,14 @@ from distutils.core import setup
 from setuptools import find_packages
 
 setup(name="xbase_util",
-      version="0.0.8",
+      version="0.1.0",
       description="网络安全基础工具",
       long_description="包含提取，预测，训练的基础工具",
       author="xyt",
       author_email="2506564278@qq.com",
       license="<MIT License>",
       packages=find_packages(),
+      url="https://gitee.com/jimonik/xbase_util.git",
       install_requires=[
       ],
       zip_safe=False,

xbase_util-0.1.0/xbase_util/es_db_util.py

@@ -0,0 +1,29 @@
+import os.path
+
+
+class EsDb:
+    def __init__(self, req):
+        self.req = req
+        self.internals = {}
+        print("初始化:Elasticsearch DB")
+
+    def get_file_by_file_id(self, node, num, prefix=None):
+        key = f'{node}!{num}'
+        if key in self.internals:
+            return self.internals[key]
+        res = self.req.search_file(f"{node}-{num}")
+        hits = res['hits']['hits']
+        if len(hits) > 0:
+            self.internals[key] = hits[0]['_source']
+            file = hits[0]['_source']
+            if prefix is None:
+                return file
+            prefix_res = prefix
+            if not prefix.endswith('/'):
+                prefix_res = f"{prefix}/"
+            origin_path = file['name']
+            basename = os.path.basename(origin_path)
+            result_path = f"{prefix_res}{basename}"
+            file['name'] = result_path
+            return file
+        return None

xbase_util-0.1.0/xbase_util/esreq.py

@@ -0,0 +1,26 @@
+import requests
+
+
+class EsReq:
+    def __init__(self, url,timeout=120):
+        self.es_url = url
+        self.timeout = timeout
+        print("初始化自定义es请求类")
+
+    def clear_all_scroll(self):
+        return requests.delete(self.es_url + "/_search/scroll", timeout=self.timeout, json={'scroll_id': '_all'})
+
+    def search(self, body, scroll):
+        requests.post(self.es_url + "/_search/scroll", data=body, timeout=self.timeout, json={'scroll_id': scroll})
+
+    def start_scroll(self, exp, scroll):
+        return requests.post(self.es_url + "/_search/scroll", timeout=self.timeout,
+                             json=exp)
+
+    def scroll_by_id(self, scroll_id, scroll):
+        return requests.post(self.es_url + "/_search/scroll", timeout=self.timeout,
+                             json={'scroll_id': scroll_id, 'scroll': scroll})
+
+    def search_file(self, id):
+        return requests.get(f"{self.es_url}/arkime_files_v30/_search", timeout=self.timeout,
+                            json={"query": {"term": {"_id": id}}})

xbase_util-0.1.0/xbase_util/handle_features_util.py

@@ -0,0 +1,161 @@
+import json
+import re
+import traceback
+from urllib.parse import unquote
+
+import pandas as pd
+
+
+def handle_uri(data):
+    print(f"处理URI:{len(data)}")
+    # 定义正则表达式，确保精确匹配各种攻击特征
+    regex_patterns = {
+        "sql": re.compile(
+            r"\b(select|union|insert|update|delete|drop|--|#| or |' or '|information_schema|database\(\)|version\(\))\b",
+            re.IGNORECASE),
+        "xss": re.compile(r"(<script\b|javascript:|onload=|onclick=|<iframe\b|src=)", re.IGNORECASE),
+        "cmd": re.compile(
+            r"(/etc/passwd\b|/etc/shadow\b|;|&&|\||\$\(.+\)|\bcurl\b|\bwget\b|\bexec\b|\bsystem\b|cmd=|proc/self/environ)",
+            re.IGNORECASE),
+        "path": re.compile(r"(\.\./|\.\.%2f|\.\.%5c|\.\.\\|\.\.;|%2f%2e%2e%2f)", re.IGNORECASE),
+        "redirect": re.compile(r"(redirect=|url=|next=|redirect_uri=|redirect:|RedirectTo=)", re.IGNORECASE),
+        "danger": re.compile(
+            r"(%3C|%3E|%27|%22|%00|%2F|%5C|%3B|%7C|%28|%29|%20|%3D|%3A|%3F|%26|%23|%2B|%25|file://|<foo|xmlns:|/etc/passwd|windows/win\.ini)",
+            re.IGNORECASE),
+        "suspicious_ext": re.compile(
+            r"\.(exe|sh|py|pl|bak|php5|jspx|bat|cmd|pif|js|vbs|vbe|sct|ini|inf|tmp|swp|jar|java|class|ps1)\b",
+            re.IGNORECASE)
+    }
+
+    # 定义多层解码函数，确保完全解码 URI
+    def fully_decode_uri(uri):
+        try:
+            decoded_uri = str(uri)
+            for _ in range(3):  # 尝试多次解码嵌套的编码
+                decoded_uri = unquote(decoded_uri)
+            return decoded_uri
+        except Exception as e:
+            return uri
+
+    def process_row(row):
+        uris = row['http.uri']
+        if not isinstance(uris, list):
+            try:
+                uris = json.loads(uris)
+                if not isinstance(uris, list):
+                    uris = [str(uris)]
+            except Exception:
+                uris = [str(uris)]
+        try:
+            decoded_uris = [fully_decode_uri(uri) for uri in uris]
+        except Exception as e:
+            traceback.print_exc()
+            exit(0)
+
+
+        # 初始化统计变量
+        param_count = 0
+        path_depth = 0
+        param_lengths = []
+        feature_flags = {key: False for key in regex_patterns.keys()}
+
+        # 遍历解码后的 URI
+        for uri in decoded_uris:
+            param_count += uri.count('&') + 1
+            path_depth += uri.count('/')
+
+            # 提取参数长度
+            if '?' in uri:
+                params = uri.split('?', 1)[-1].split('&')
+                for param in params:
+                    if '=' in param:
+                        _, value = param.split('=', 1)
+                        param_lengths.append(len(value))
+
+            # 检查正则匹配特征
+            for key, pattern in regex_patterns.items():
+                if pattern.search(uri):
+                    feature_flags[key] = True
+
+        # 计算参数长度的统计值
+        avg_length = sum(param_lengths) / len(param_lengths) if param_lengths else 0
+        max_length = max(param_lengths) if param_lengths else 0
+
+        # 创建返回结果字典
+        result = {
+            "URI_FEATURES_EXTRA_param_count": param_count,
+            "URI_FEATURES_EXTRA_path_depth": path_depth,
+            "URI_FEATURES_EXTRA_param_length_avg": avg_length,
+            "URI_FEATURES_EXTRA_param_length_max": max_length,
+        }
+
+        # 添加特征标志到结果
+        for key, value in feature_flags.items():
+            result[f"URI_FEATURES_EXTRA_contains_{key}"] = value
+
+        return result
+    feature_data = data.progress_apply(process_row, axis=1, result_type="expand")
+    data = pd.concat([data, feature_data], axis=1)
+    return data
+
+
+def handle_ua(data):
+    print("处理UA")
+    data['http.useragent'] = data['http.useragent'].fillna('').astype(str)
+    # 处理换行符及多余空格
+    data['http.useragent'] = data['http.useragent'].str.replace(r'\s+', ' ', regex=True)
+    # 常见攻击的 User-Agent 字符串匹配模式，忽略大小写
+    attack_patterns = '|'.join([
+        r"\bselect\b", r"\bunion\b", r"\binsert\b", r"\bupdate\b", r"\bdelete\b", r"\bdrop\b", r"--", r"#", r" or ",
+        r"' or '",
+        r"information_schema", r"database\(\)", r"version\(\)",  # SQL注入相关
+        r"<script>", r"javascript:", r"onload=", r"onclick=", r"<iframe>", r"src=",  # XSS相关
+        r"/etc/passwd", r"/etc/shadow", r"\&\&", r"\|", r"\$\(\)", r"exec", r"system",  # 命令执行相关
+        r"\.\./", r"\.\.%2f", r"\.\.%5c", r"%c0%af", r"%252e%252e%252f",  # 路径遍历
+        r"\.php", r"\.asp", r"\.jsp", r"\.exe", r"\.sh", r"\.py", r"\.pl",  # 文件扩展名
+        r"redirect=", r"url=", r"next=",  # 重定向
+        r"%3C", r"%3E", r"%27", r"%22", r"%00", r"%2F", r"%5C", r"%3B", r"%7C", r"%2E", r"%28", r"%29",  # 编码
+        r'Googlebot', r'Bingbot', r'Slurp', r'curl', r'wget', r'Nmap',
+        r'SQLMap', r'Nikto', r'Dirbuster', r'python-requests', r'Apache-HttpClient',
+        r'Postman', r'Burp Suite', r'Fuzzing', r'nessus'
+    ])
+    # 企业客户端 User-Agent 模式
+    enterprise_patterns = '|'.join([
+        r'MicroMessenger', r'wxwork', r'QQ/', r'QQBrowser', r'Alipay', r'UCWEB'
+    ])
+    # 批量检查是否为攻击的 User-Agent，忽略大小写
+    data['UserAgent_is_attack'] = data['http.useragent'].str.contains(attack_patterns, case=False, regex=True)
+    # 批量检查是否为企业客户端，忽略大小写
+    data['UserAgent_is_enterprise'] = data['http.useragent'].str.contains(enterprise_patterns, case=False)
+    # 提取浏览器和版本
+    data['UserAgent_browser'] = data['http.useragent'].str.extract(r'(Chrome|Firefox|Safari|MSIE|Edge|Opera|Trident)',
+                                                                   expand=False, flags=re.IGNORECASE).fillna("Unknown")
+    data['UserAgent_browser_version'] = data['http.useragent'].str.extract(
+        r'Chrome/([\d\.]+)|Firefox/([\d\.]+)|Version/([\d\.]+).*Safari|MSIE ([\d\.]+)|Edge/([\d\.]+)|Opera/([\d\.]+)|Trident/([\d\.]+)',
+        expand=False, flags=re.IGNORECASE).bfill(axis=1).fillna("Unknown").iloc[:, 0]
+    # 提取操作系统和版本
+    os_info = data['http.useragent'].str.extract(
+        r'(Windows NT [\d\.]+|Mac OS X [\d_\.]+|Linux|Android [\d\.]+|iOS [\d_\.]+|Ubuntu|Debian|CentOS|Red Hat)',
+        expand=False, flags=re.IGNORECASE)
+    data['UserAgent_os'] = os_info.str.extract(r'(Windows|Mac OS X|Linux|Android|iOS|Ubuntu|Debian|CentOS|Red Hat)',
+                                               expand=False, flags=re.IGNORECASE).fillna("Unknown")
+    data['UserAgent_os_version'] = os_info.str.extract(r'([\d\._]+)', expand=False).fillna("Unknown")
+    # 提取设备类型，忽略大小写
+    data['UserAgent_device_type'] = data['http.useragent'].str.contains('mobile|android|iphone', case=False).map(
+        {True: 'Mobile', False: 'Desktop'})
+    # 提取硬件平台，增加对 x64 的匹配
+    data['UserAgent_platform'] = data['http.useragent'].str.extract(r'(x86|x86_64|arm|arm64|x64)', expand=False,
+                                                                    flags=re.IGNORECASE).fillna('Unknown')
+    # 判断是否为爬虫，忽略大小写
+    data['UserAgent_is_bot'] = data['http.useragent'].str.contains('bot|crawler|spider|slurp|curl|wget|httpclient',
+                                                                   case=False)
+    # 提取语言偏好（如果存在），忽略大小写
+    data['UserAgent_language'] = data['http.useragent'].str.extract(r'\b([a-z]{2}-[A-Z]{2})\b', expand=False,
+                                                                    flags=re.IGNORECASE).fillna("Unknown")
+    # 统计 User-Agent 中的特殊字符个数
+    data['UserAgent_special_char_count'] = data['http.useragent'].progress_apply(
+        lambda x: len(re.findall(r'[!@#$%^&*\'=:|{}]', x, flags=re.IGNORECASE)))
+    # 更新 UserAgent_is_unknown 的计算逻辑
+    data['UserAgent_is_unknown'] = data[['UserAgent_browser', 'UserAgent_os', 'UserAgent_platform']].isna().any(
+        axis=1).fillna("Unknown")
+    return data

xbase_util-0.1.0/xbase_util/pcap_util.py

@@ -0,0 +1,247 @@
+import math
+import os
+import struct
+import time
+import zlib
+from datetime import datetime
+
+from Crypto.Cipher import AES
+from zstandard import ZstdDecompressor
+
+
+def fix_pos(pos, packetPosEncoding):
+    if pos is None or len(pos) == 0:
+        return
+    if packetPosEncoding == "gap0":
+        last = 0
+        lastgap = 0
+        for i, pos_item in enumerate(pos):
+            if pos[i] < 0:
+                last = 0
+            else:
+                if pos[i] == 0:
+                    pos[i] = last + lastgap
+                else:
+                    lastgap = pos[i]
+                    pos[i] += last
+                last = pos[i]
+
+
+def group_numbers(nums):
+    result = []
+    for num in nums:
+        if num < 0:
+            result.append([num])
+        elif result:
+            result[-1].append(num)
+    return result
+
+
+def decompress_streaming(compressed_data, id, fro):
+    try:
+        decompressor = ZstdDecompressor()
+        with decompressor.stream_reader(compressed_data) as reader:
+            decompressed_data = reader.read()
+            return decompressed_data
+    except Exception as e:
+        print(f"解码错误：{e}  {id}")
+        return bytearray()
+
+
+def read_header(param_map, id):
+    shortHeader = None
+    headBuffer = os.read(param_map['fd'], 64)
+    if param_map['encoding'] == 'aes-256-ctr':
+        if 'iv' in param_map:
+            param_map['iv'][12:16] = struct.pack('>I', 0)
+            headBuffer = bytearray(
+                AES.new(param_map['encKey'], AES.MODE_CTR, nonce=param_map['iv']).decrypt(bytes(headBuffer)))
+        else:
+            print("读取头部信息失败，iv向量为空")
+    elif param_map['encoding'] == 'xor-2048':
+        for i in range(len(headBuffer)):
+            headBuffer[i] ^= param_map['encKey'][i % 256]
+    if param_map['uncompressedBits']:
+        if param_map['compression'] == 'gzip':
+            headBuffer = zlib.decompress(bytes(headBuffer), zlib.MAX_WBITS | 16)
+        elif param_map['compression'] == 'zstd':
+            headBuffer = decompress_streaming(headBuffer, id, "header")
+    headBuffer = headBuffer[:24]
+    magic = struct.unpack('<I', headBuffer[:4])[0]
+    bigEndian = (magic == 0xd4c3b2a1 or magic == 0x4d3cb2a1)
+    # nanosecond = (magic == 0xa1b23c4d or magic == 0x4d3cb2a1)
+    if not bigEndian and magic not in {0xa1b2c3d4, 0xa1b23c4d, 0xa1b2c3d5}:
+        corrupt = True
+        # os.close(param_map['fd'])
+        raise ValueError("Corrupt PCAP header")
+    if magic == 0xa1b2c3d5:
+        shortHeader = struct.unpack('<I', headBuffer[8:12])[0]
+        headBuffer[0] = 0xd4  # Reset header to normal
+    linkType = struct.unpack('>I' if bigEndian else '<I', headBuffer[20:24])[0]
+    return headBuffer, shortHeader, bigEndian
+
+
+def create_decipher(pos, param_map):
+    param_map['iv'][12:16] = struct.pack('>I', pos)
+    return AES.new(param_map['encKey'], AES.MODE_CTR, nonce=param_map['iv'])
+
+
+def read_packet_internal(pos_arg, hp_len_arg, param_map, id):
+    pos = pos_arg
+    hp_len = hp_len_arg
+    if hp_len == -1:
+        if param_map['compression'] == "zstd":
+            hp_len = param_map['uncompressedBitsSize']
+        else:
+            hp_len = 2048
+    inside_offset = 0
+    if param_map['uncompressedBits']:
+        inside_offset = pos & param_map['uncompressedBitsSize'] - 1
+        pos = math.floor(pos / param_map['uncompressedBitsSize'])
+    pos_offset = 0
+    if param_map['encoding'] == 'aes-256-ctr':
+        pos_offset = pos % 16
+        pos = pos - pos_offset
+    elif param_map['encoding'] == 'xor-2048':
+        pos_offset = pos % 256
+        pos = pos - pos_offset
+
+    hp_len = 256 * math.ceil((hp_len + inside_offset + pos_offset) / 256)
+    buffer = bytearray(hp_len)
+    os.lseek(param_map['fd'], pos, os.SEEK_SET)
+    read_buffer = os.read(param_map['fd'], len(buffer))
+    if len(read_buffer) - pos_offset < 16:
+        return None
+    if param_map['encoding'] == 'aes-256-ctr':
+        decipher = create_decipher(pos // 16, param_map)
+        read_buffer = bytearray(decipher.decrypt(read_buffer))[pos_offset:]
+    elif param_map['encoding'] == 'xor-2048':
+        read_buffer = bytearray(b ^ param_map['encKey'][i % 256] for i, b in enumerate(read_buffer))[pos_offset:]
+    if param_map['uncompressedBits']:
+        try:
+            if param_map['compression'] == 'gzip':
+                read_buffer = zlib.decompress(read_buffer, zlib.MAX_WBITS | 16)
+            elif param_map['compression'] == 'zstd':
+                read_buffer = decompress_streaming(read_buffer, id, "packet")
+        except Exception as e:
+            print(f"PCAP uncompress issue:  {pos} {len(buffer)} {read_buffer} {e}")
+            return None
+    if inside_offset:
+        read_buffer = read_buffer[inside_offset:]
+    header_len = 16 if param_map['shortHeader'] is None else 6
+    if len(read_buffer) < header_len:
+        if hp_len_arg == -1 and param_map['compression'] == 'zstd':
+            return read_packet_internal(pos_arg, param_map['uncompressedBitsSize'] * 2, param_map, id)
+        print(f"Not enough data {len(read_buffer)} for header {header_len}")
+        return None
+    packet_len = struct.unpack('>I' if param_map['bigEndian'] else '<I', read_buffer[8:12])[
+        0] if param_map['shortHeader'] is None else \
+        struct.unpack('>H' if param_map['bigEndian'] else '<H', read_buffer[:2])[0]
+    if packet_len < 0 or packet_len > 0xffff:
+        return None
+    if header_len + packet_len <= len(read_buffer):
+        if param_map['shortHeader'] is not None:
+            t = struct.unpack('<I', read_buffer[2:6])[0]
+            sec = (t >> 20) + param_map['shortHeader']
+            usec = t & 0xfffff
+            new_buffer = bytearray(16 + packet_len)
+            struct.pack_into('<I', new_buffer, 0, sec)
+            struct.pack_into('<I', new_buffer, 4, usec)
+            struct.pack_into('<I', new_buffer, 8, packet_len)
+            struct.pack_into('<I', new_buffer, 12, packet_len)
+            new_buffer[16:] = read_buffer[6:packet_len + 6]
+            return new_buffer
+        return read_buffer[:header_len + packet_len]
+
+    if hp_len_arg != -1:
+        return None
+
+    return read_packet_internal(pos_arg, 16 + packet_len, param_map, id)
+
+
+def read_packet(pos, param_map, id):
+    if 'fd' not in param_map or not param_map['fd']:
+        time.sleep(0.01)
+        return read_packet(pos, param_map['fd'], id)
+    return read_packet_internal(pos, -1, param_map, id)
+
+
+def get_file_and_read_pos(id, file, pos_list):
+    filename = file['name']
+    if not os.path.isfile(filename):
+        print(f"文件不存在:{filename}")
+        return None
+    encoding = file.get('encoding', 'normal')
+    encKey = None
+    iv = None
+    compression = None
+    if 'dek' in file:
+        dek = bytes.fromhex(file['dek'])
+        encKey = AES.new(file['kek'].encode(), AES.MODE_CBC).decrypt(dek)
+
+    if 'uncompressedBits' in file:
+        uncompressedBits = file['uncompressedBits']
+        uncompressedBitsSize = 2 ** uncompressedBits
+        compression = 'gzip'
+    else:
+        uncompressedBits = None
+        uncompressedBitsSize = 0
+    if 'compression' in file:
+        compression = file['compression']
+
+    if 'iv' in file:
+        iv_ = bytes.fromhex(file['iv'])
+        iv = bytearray(16)
+        iv[:len(iv_)] = iv_
+    fd = os.open(filename, os.O_RDONLY)
+    param_map = {
+        "fd": fd,
+        "encoding": encoding,
+        "iv": iv,
+        "encKey": encKey,
+        "uncompressedBits": uncompressedBits,
+        "compression": compression,
+        "uncompressedBitsSize": uncompressedBitsSize
+    }
+    res = bytearray()
+    headBuffer, shortHeader, bigEndian = read_header(param_map, id)
+    res.extend(headBuffer)
+    param_map['shortHeader'] = shortHeader
+    param_map['bigEndian'] = bigEndian
+    # _________________________________
+    byte_array = bytearray(0xfffe)
+    next_packet = 0
+    b_offset = 0
+    packets = {}
+    i = 0
+    for pos in pos_list:
+        packet_bytes = read_packet(pos, param_map, id)
+        if not packet_bytes:
+            continue
+        packets[i] = packet_bytes
+        while next_packet in packets:
+            buffer = packets[next_packet]
+            del packets[next_packet]
+            next_packet = next_packet + 1
+            if b_offset + len(buffer) > len(byte_array):
+                res.extend(byte_array[:b_offset])
+                b_offset = 0
+                byte_array = bytearray(0xfffe)
+            byte_array[b_offset:b_offset + len(buffer)] = buffer
+            b_offset += len(buffer)
+        i = i + 1
+    os.close(fd)
+    res.extend(byte_array[:b_offset])
+    return res
+
+
+def process_session_id_disk_simple(id, node, packet_pos, esdb, pcap_path_prefix):
+    packetPos = packet_pos
+    file = esdb.get_file_by_file_id(node=node, num=abs(packetPos[0]),
+                                    prefix=None if pcap_path_prefix == "origin" else pcap_path_prefix)
+    if file is None:
+        return None
+    fix_pos(packetPos, file['packetPosEncoding'])
+    pos_list = group_numbers(packetPos)[0]
+    pos_list.pop(0)
+    return get_file_and_read_pos(id, file, pos_list)

{xbase_util-0.0.8 → xbase_util-0.1.0}/xbase_util.egg-info/PKG-INFO

@@ -1,7 +1,8 @@
 Metadata-Version: 2.1
 Name: xbase-util
-Version: 0.0.8
+Version: 0.1.0
 Summary: 网络安全基础工具
+Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt
 Author-email: 2506564278@qq.com
 License: <MIT License>

{xbase_util-0.0.8 → xbase_util-0.1.0}/xbase_util.egg-info/SOURCES.txt

@@ -1,6 +1,10 @@
 README.md
 setup.py
 xbase_util/__init__.py
+xbase_util/es_db_util.py
+xbase_util/esreq.py
+xbase_util/handle_features_util.py
+xbase_util/pcap_util.py
 xbase_util/xbase_util.py
 xbase_util.egg-info/PKG-INFO
 xbase_util.egg-info/SOURCES.txt