xbase-util 0.4.3__tar.gz → 0.4.5__tar.gz

{xbase_util-0.4.3 → xbase_util-0.4.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase_util
-Version: 0.4.3
+Version: 0.4.5
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt

{xbase_util-0.4.3 → xbase_util-0.4.5}/setup.py

@@ -3,7 +3,7 @@ from distutils.core import setup
 from setuptools import find_packages
 
 setup(name="xbase_util",
-      version="0.4.3",
+      version="0.4.5",
       description="网络安全基础工具",
       long_description="包含提取，预测，训练的基础工具",
       author="xyt",
@@ -12,6 +12,7 @@ setup(name="xbase_util",
       packages=find_packages(),
       url="https://gitee.com/jimonik/xbase_util.git",
       install_requires=[
+
       ],
       zip_safe=False,
       package_data={

xbase_util-0.4.5/xbase_util/dangerous_util.py

@@ -0,0 +1,101 @@
+import splunklib.client as client
+from splunklib import results
+
+
+def get_splunk_pa(start_time, end_time, splunk_host,
+                  splunk_port,
+                  splunk_username,
+                  splunk_password,
+                  splunk_scheme="https",
+                  splunk_filter="THREAT AND NOT informational"):
+    """
+    获取PA威胁信息
+    :param splunk_filter:
+    :param start_time:
+    :param end_time:
+    :param splunk_host:
+    :param splunk_port:
+    :param splunk_username:
+    :param splunk_password:
+    :param splunk_scheme:
+    :return:
+    """
+    service = client.connect(
+        host=splunk_host,
+        port=splunk_port,
+        scheme=splunk_scheme,
+        username=splunk_username,
+        password=splunk_password
+    )
+    job = service.jobs.oneshot(
+        """search index=idx_pa 
+FILTER_TEXT
+| rex field=_raw  "THREAT,(?P<LOG_TYPE>.+?),.*?,(?P<PA_DATE>.*?),(?P<SIP>.*?),(?P<DIP>.*?),(?:.*?,.*?,.*?){7},(?P<S_PORT>.*?),(?P<D_PORT>.*?),.*?,.*?,.*?,(?P<PROTOCOL>.*?),(?P<DENY_METHOD>.*?),(?P<THREAT_SUMMARY>.*?),(?P<SEVERITY>medium|high|critical|low),"  
+| eval RAW_BAK=_raw 
+| eval THREAT_TIME = strftime(strptime(PA_DATE, "%Y/%m/%d %H:%M:%S"), "%Y-%m-%d %H:%M:%S")
+| rex mode=sed field=_raw "s/.*?,\w{8}-\w{4}-\w{4}-\w{4}-\w{12},/start_ama_flag,/g"
+| rex field=_raw "start_ama_flag,.*?,.*?,(?<XFF_IP>.*?),"
+| eval _raw=RAW_BAK
+| table THREAT_TIME,SIP,S_PORT, DIP, D_PORT,XFF_IP,PROTOCOL, DENY_METHOD, THREAT_SUMMARY, SEVERITY
+| dedup THREAT_TIME,SIP,S_PORT, DIP, D_PORT,XFF_IP,PROTOCOL
+            """.replace("FILTER_TEXT", splunk_filter), **{
+            "earliest_time": start_time.strftime('%Y-%m-%dT%H:%M:%S'),
+            "latest_time": end_time.strftime('%Y-%m-%dT%H:%M:%S'),
+            "output_mode": "json",
+            "count": 20000
+        })
+    return [item for item in results.JSONResultsReader(job) if isinstance(item, dict)]
+
+
+def get_splunk_waf(start_time, end_time, splunk_host,
+                   splunk_port,
+                   splunk_username,
+                   splunk_password,
+                   splunk_scheme="https"):
+    # splunk里面用这个
+    """
+sourcetype=changting:waf
+| rex field=_raw (?P<THREAT_TIME>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+\d{2}:\d{2})
+| rex field=_raw "\"src_ip\":\"(?<SIP>[^\"]+)\""
+| rex field=_raw "\"protocol\":\"(?<PROTOCOL>[^\"]+)\""
+| rex field=_raw "\"src_port\":(?<S_PORT>\d+)"
+| rex field=_raw "\"dest_ip\":\"(?<DIP>[^\"]+)\""
+| rex field=_raw "\"dest_port\":(?<D_PORT>\d+)"
+| rex field=_raw "\"risk_level\":\"(?<SEVERITY>[^\"]+)\""
+| rex field=_raw "\"action\":\"(?<DENY_METHOD>[^\"]+)\""
+| rex field=_raw "\"reason\":\"(?<THREAT_SUMMARY>[^\"]+)\""
+| rex field=_raw "\"x_forwarded_for\":\"(?<XFF_IP>[^\"]+)\""
+| table THREAT_TIME,SIP,S_PORT,DIP,D_PORT,XFF_IP,PROTOCOL,DENY_METHOD,THREAT_SUMMARY,SEVERITY
+| dedup THREAT_TIME,SIP,S_PORT,DIP,D_PORT,XFF_IP,PROTOCOL
+    """
+    service = client.connect(
+        host=splunk_host,
+        port=splunk_port,
+        scheme=splunk_scheme,
+        username=splunk_username,
+        password=splunk_password
+    )
+    # | rex field=_raw "\\"src_port\\":(?<S_PORT>\d+)"
+    # | rex field=_raw "\\"dest_port\\":(?<D_PORT>\d+)"
+    exp = """search sourcetype=changting:waf
+| rex field=_raw "(?P<THREAT_TIME>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\+\\d{2}:\\d{2})"
+| rex field=_raw "\\"dest_ip\\":\\"(?<DIP>[^\\"]+)\\"" 
+| rex field=_raw "\\"src_ip\\":\\"(?<SIP>[^\\"]+)\\""
+| rex field=_raw "\\"src_port\\":(?<S_PORT>\\d+)"
+| rex field=_raw "\\"dest_port\\":(?<D_PORT>\\d+)"
+| rex field=_raw "\\"protocol\\":\\"(?<PROTOCOL>[^\\"]+)\\""
+| rex field=_raw "\\"x_forwarded_for\\":\\"(?<XFF_IP>[^\\"]+)\\""
+| rex field=_raw "\\"action\\":\\"(?<DENY_METHOD>[^\\"]+)\\""
+| rex field=_raw "\\"reason\\":\\"(?<THREAT_SUMMARY>[^\\"]+)\\""
+| rex field=_raw "\\"risk_level\\":\\"(?<SEVERITY>[^\\"]+)\\""
+| dedup THREAT_TIME,SIP,S_PORT,DIP,D_PORT,XFF_IP,PROTOCOL
+| table THREAT_TIME,SIP,S_PORT,DIP,D_PORT,XFF_IP,PROTOCOL,DENY_METHOD,THREAT_SUMMARY,SEVERITY
+        """
+    job = service.jobs.oneshot(
+        exp, **{
+            "earliest_time": start_time.strftime('%Y-%m-%dT%H:%M:%S'),
+            "latest_time": end_time.strftime('%Y-%m-%dT%H:%M:%S'),
+            "output_mode": "json",
+            "count": 20000
+        })
+    return [item for item in results.JSONResultsReader(job) if isinstance(item, dict)]

{xbase_util-0.4.3 → xbase_util-0.4.5}/xbase_util/es_db_util.py

@@ -5,7 +5,6 @@ class EsDb:
     def __init__(self, req, manager):
         self.req = req
         self.internals = manager.dict()
-        print("初始化:Elasticsearch DB")
 
     def get_file_by_file_id(self, node, num, prefix=None):
         key = f'{node}!{num}'

{xbase_util-0.4.3 → xbase_util-0.4.5}/xbase_util/xbase_util.py

@@ -1,6 +1,9 @@
 import json
+import logging
 import os
 import re
+from datetime import datetime
+from logging.handlers import TimedRotatingFileHandler
 from urllib.parse import urlparse, parse_qs
 
 import execjs
@@ -427,3 +430,28 @@ def get_project_root(project_name):
             return f"{current_directory}{os.sep}"
         current_directory = os.path.dirname(current_directory)
     return None
+
+
+def setup_logger(process_name, dir):
+    logger = logging.getLogger(process_name)
+    logger.setLevel(logging.DEBUG)  # 设置日志级别
+    log_filename = check_path(f"{dir}/{process_name}.log")
+    handler = TimedRotatingFileHandler(
+        log_filename,
+        when="midnight",
+        interval=1,
+        backupCount=30,
+        encoding="utf-8"
+    )
+
+    def custom_namer(filename):
+        base, ext = os.path.splitext(os.path.splitext(filename)[0])
+        current_time = datetime.now().strftime("_%Y_%m_%d")
+        return f"{base}{current_time}{ext}"
+
+    handler.namer = custom_namer
+    handler.setLevel(logging.DEBUG)
+    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+    return logger

{xbase_util-0.4.3 → xbase_util-0.4.5}/xbase_util.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase-util
-Version: 0.4.3
+Version: 0.4.5
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt

{xbase_util-0.4.3 → xbase_util-0.4.5}/xbase_util.egg-info/SOURCES.txt

@@ -2,6 +2,7 @@ README.md
 setup.py
 xbase_util/__init__.py
 xbase_util/add_column_util.py
+xbase_util/dangerous_util.py
 xbase_util/es_db_util.py
 xbase_util/esreq.py
 xbase_util/geo_util.py