xbase-util 0.1.3__tar.gz → 0.1.5__tar.gz

{xbase_util-0.1.3 → xbase_util-0.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase_util
-Version: 0.1.3
+Version: 0.1.5
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt

{xbase_util-0.1.3 → xbase_util-0.1.5}/setup.py

@@ -3,7 +3,7 @@ from distutils.core import setup
 from setuptools import find_packages
 
 setup(name="xbase_util",
-      version="0.1.3",
+      version="0.1.5",
       description="网络安全基础工具",
       long_description="包含提取，预测，训练的基础工具",
       author="xyt",

xbase_util-0.1.5/xbase_util/db/bean/ConfigBean.py

@@ -0,0 +1,103 @@
+from sqlalchemy import Column, Integer, String, TEXT, Boolean
+
+from xbase_util.db.bean import DbBase
+
+
+class ConfigBean(DbBase):
+    __tablename__ = 'configs'
+    id = Column(Integer, primary_key=True)
+    description = Column(TEXT)
+
+    label_all_true = Column(Boolean, nullable=False)
+    label_is_output_unmatch = Column(Boolean, nullable=False)
+    label_duration = Column(Integer, nullable=False)
+    splitNumber = Column(Integer, nullable=False)
+
+    mapping_le_path = Column(String, nullable=False)
+    pcap_per_subsection = Column(Integer, nullable=False)
+    pcap_process = Column(Integer, nullable=False)
+    pcap_thread_in_process = Column(Integer, nullable=False)
+    replace_source = Column(TEXT, nullable=False)  #用列表传
+    replace_destination = Column(String, nullable=False)
+    replace_mapping = Column(TEXT, nullable=False)  #用列表传
+    replace_save_to = Column(String, nullable=False)
+
+    session_all_true = Column(Boolean, nullable=False)
+    session_start_time = Column(String, nullable=False)
+    session_end_time = Column(String, nullable=False)
+    session_expression = Column(String)
+    session_alive = Column(String)
+    catalogue = Column(String)
+    session_size = Column(Integer)
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'description': self.description or "",
+
+            'label_all_true': self.label_all_true,
+            'label_is_output_unmatch': self.label_is_output_unmatch,
+            'label_duration': self.label_duration,
+            'splitNumber': self.splitNumber,
+
+            'mapping_le_path': self.mapping_le_path or "",
+
+            'pcap_per_subsection': self.pcap_per_subsection,
+            'pcap_process': self.pcap_process,
+            'pcap_thread_in_process': self.pcap_thread_in_process,
+
+            'replace_source': self.replace_source or "",
+            'replace_destination': self.replace_destination or "",
+            'replace_mapping': self.replace_mapping or "",
+            'replace_save_to': self.replace_save_to or "",
+
+            'session_all_true': self.session_all_true,
+            'session_start_time': self.session_start_time or "",
+            'session_end_time': self.session_end_time or "",
+            'session_expression': self.session_expression or "",
+            'session_alive': self.session_alive or "",
+            'session_size': self.session_size or "",
+            'catalogue': self.catalogue or "",
+        }
+
+    def to_session_dict(self):
+        return {
+            'id': self.id,
+            'session_all_true': self.session_all_true,
+            'session_start_time': self.session_start_time or "",
+            'session_end_time': self.session_end_time or "",
+            'session_expression': self.session_expression or "",
+            'session_alive': self.session_alive or "",
+            'session_size': self.session_size or "",
+        }
+
+    def to_pcap_dict(self):
+        return {
+            'id': self.id,
+            'pcap_per_subsection': self.pcap_per_subsection,
+            'pcap_process': self.pcap_process,
+            'pcap_thread_in_process': self.pcap_thread_in_process,
+        }
+
+    def to_label_dict(self):
+        return {
+            'id': self.id,
+            'label_all_true': self.label_all_true,
+            'label_is_output_unmatch': self.label_is_output_unmatch,
+            'label_duration': self.label_duration,
+        }
+
+    def to_mapping(self):
+        return {
+            'id': self.id,
+            'mapping_le_path': self.mapping_le_path or "",
+        }
+
+    def to_replace(self):
+        return {
+            'id': self.id,
+            'replace_source': self.replace_source or "",
+            'replace_destination': self.replace_destination or "",
+            'replace_mapping': self.replace_mapping or "",
+            'replace_save_to': self.replace_save_to or "",
+        }

xbase_util-0.1.5/xbase_util/db/bean/CurrentConfigBean.py

@@ -0,0 +1,10 @@
+from sqlalchemy import Column, Integer,String
+
+from xbase_util.db.bean import DbBase
+
+
+class CurrentConfig(DbBase):
+    __tablename__ = 'currentconfig'
+    id = Column(Integer, primary_key=True)
+    config_id = Column(Integer)
+    description = Column(String)

xbase_util-0.1.5/xbase_util/db/bean/FlowBean.py

@@ -0,0 +1,10 @@
+from sqlalchemy import Column, Integer, TEXT
+
+from xbase_util.db.bean import DbBase
+
+
+class FlowBean(DbBase):
+    __tablename__ = 'flows'
+    id = Column(Integer, primary_key=True)
+    description = Column(TEXT)
+    step = Column(TEXT)

xbase_util-0.1.5/xbase_util/db/bean/TaskTemplateBean.py

@@ -0,0 +1,28 @@
+from sqlalchemy import Column, Integer, String, Boolean
+
+from xbase_util.db.bean import DbBase
+
+
+class TaskTemplateBean(DbBase):
+    __tablename__ = 'tasktemplatebean'
+    id = Column(Integer, primary_key=True)
+
+    config_id = Column(String)
+    flow_id = Column(String)
+    description = Column(String)
+
+    is_scheduled = Column(Boolean, default=False)  # 是否为定时任务
+    scheduled_start_time = Column(String, nullable=True)  # 定时任务的开始时间
+    scheduled_interval_minutes = Column(Integer, nullable=True)  # 定时任务的执行间隔（以分钟为单位）
+    scheduled_period_minutes = Column(Integer, nullable=True)  # 要获取的时间段（以分钟为单位）
+
+    def to_dict(self):
+        return {
+            "id": self.id,
+            "config_id": self.config_id,
+            "flow_id": self.flow_id,
+            "description": self.description,
+            "is_scheduled": self.is_scheduled,
+            "start_time": self.scheduled_start_time,
+            "interval_minutes": self.scheduled_interval_minutes,
+        }

xbase_util-0.1.5/xbase_util/db/bean/__init__.py

@@ -0,0 +1,3 @@
+from sqlalchemy.orm import declarative_base
+
+DbBase = declarative_base()

xbase_util-0.1.5/xbase_util/db/dao/ConfigDao.py

@@ -0,0 +1,172 @@
+import traceback
+
+from xbase_util.db.bean.ConfigBean import ConfigBean
+
+
+class ConfigDao:
+    def __init__(self,Session):
+        self.Session = Session
+
+    def add(self, id, description, label_all_true, label_is_output_unmatch,
+            label_duration,
+            mapping_le_path, pcap_per_subsection, pcap_process, pcap_thread_in_process,
+            replace_source, replace_destination, replace_mapping, replace_save_to, session_all_true, session_start_time,
+            session_end_time, session_expression, session_alive, session_size, splitNumber,catalogue
+            ):
+        with self.Session() as session:
+            try:
+                if id is None:
+                    bean = ConfigBean(
+                        description=description,
+                        label_all_true=label_all_true,
+                        label_is_output_unmatch=label_is_output_unmatch,
+                        label_duration=label_duration,
+                        mapping_le_path=mapping_le_path,
+                        pcap_per_subsection=pcap_per_subsection,
+                        pcap_process=pcap_process,
+                        pcap_thread_in_process=pcap_thread_in_process,
+                        replace_source=replace_source,
+                        replace_destination=replace_destination,
+                        replace_mapping=replace_mapping,
+                        replace_save_to=replace_save_to,
+                        session_all_true=session_all_true,
+                        session_start_time=session_start_time,
+                        session_end_time=session_end_time,
+                        session_expression=session_expression,
+                        session_alive=session_alive,
+                        session_size=session_size,
+                        splitNumber=splitNumber,
+                        catalogue=catalogue
+                    )
+                    session.add(bean)
+                    session.commit()
+                    return True
+                else:
+                    config = session.query(ConfigBean).filter(ConfigBean.id == id).first()
+                    if config is None:
+                        return False
+                    config.description = description
+                    config.label_all_true = label_all_true
+                    config.label_is_output_unmatch = label_is_output_unmatch
+                    config.label_duration = label_duration
+                    config.mapping_le_path = mapping_le_path
+                    config.pcap_per_subsection = pcap_per_subsection
+                    config.pcap_process = pcap_process
+                    config.pcap_thread_in_process = pcap_thread_in_process
+                    config.replace_source = replace_source
+                    config.replace_destination = replace_destination
+                    config.replace_mapping = replace_mapping
+                    config.replace_save_to = replace_save_to
+                    config.session_all_true = session_all_true
+                    config.session_start_time = session_start_time
+                    config.session_end_time = session_end_time
+                    config.session_expression = session_expression
+                    config.session_alive = session_alive
+                    config.session_size = session_size
+                    config.splitNumber = splitNumber
+                    config.catalogue=catalogue
+                    session.commit()
+            except Exception as e:
+                session.rollback()
+                traceback.print_exc()
+                print(e)
+
+    def get_config_file_list(self):
+        with self.Session() as session:
+            try:
+                config_list = session.query(ConfigBean).all()
+                return [d.to_dict() for d in config_list]
+            except Exception as e:
+                session.rollback()
+                print(e)
+
+    def remove_by_id(self, id):
+        with self.Session() as session:
+            try:
+                session.query(ConfigBean).filter(ConfigBean.id == id).delete()
+                session.commit()
+            except Exception as e:
+                session.rollback()
+                print(e)
+
+    def get_config_by_id(self, id):
+        with self.Session() as session:
+            try:
+                return session.query(ConfigBean).filter(ConfigBean.id == id).first()
+            except Exception as e:
+                session.rollback()
+                print(e)
+
+    def set_config_session_by_id(self, id, session_all_true, session_start_time, session_end_time, session_expression,
+                                 session_alive, session_size):
+        with self.Session() as session:
+            try:
+                config = session.query(ConfigBean).filter(ConfigBean.id == id).first()
+                config.session_all_true = session_all_true
+                config.session_start_time = session_start_time
+                config.session_end_time = session_end_time
+                config.session_expression = session_expression
+                config.session_alive = session_alive
+                config.session_size = session_size
+                session.commit()
+            except Exception as e:
+                session.rollback()
+                print(e)
+
+    def set_config_pcap_by_id(self, id,
+                              pcap_per_subsection,
+                              pcap_process,
+                              pcap_thread_in_process):
+        with self.Session() as session:
+            try:
+                config = session.query(ConfigBean).filter(ConfigBean.id == id).first()
+                config.pcap_per_subsection = pcap_per_subsection
+                config.pcap_process = pcap_process
+                config.pcap_thread_in_process = pcap_thread_in_process
+                session.commit()
+            except Exception as e:
+                session.rollback()
+                print(e)
+
+    def set_config_label_by_id(self, id,
+                               label_all_true,
+                               label_is_output_unmatch,
+                               label_duration):
+        with self.Session() as session:
+            try:
+                config = session.query(ConfigBean).filter(ConfigBean.id == id).first()
+                config.label_all_true = label_all_true
+                config.label_is_output_unmatch = label_is_output_unmatch
+                config.label_duration = label_duration
+                session.commit()
+            except Exception as e:
+                session.rollback()
+                print(e)
+
+    def set_config_mapping_by_id(self, id, mapping_le_path):
+        with self.Session() as session:
+            try:
+                config = session.query(ConfigBean).filter(ConfigBean.id == id).first()
+                config.mapping_le_path = mapping_le_path
+                session.commit()
+            except Exception as e:
+                session.rollback()
+                print(e)
+
+    def set_config_replace_by_id(self, id, replace_source, replace_destination, replace_mapping, replace_save_to):
+        with self.Session() as session:
+            try:
+                config = session.query(ConfigBean).filter(ConfigBean.id == id).first()
+                config.replace_source = replace_source
+                config.replace_destination = replace_destination
+                config.replace_mapping = replace_mapping
+                config.replace_save_to = replace_save_to
+                session.commit()
+            except Exception as e:
+                session.rollback()
+                print(e)
+# 修改脚本带redis获取状态
+# 提异常，正常dns和异常都要，要新的黑白样本
+# app
+# capture
+# 日报规范问题 [做了什么][完成的进度][遇到的问题][问题研究的进度和方案]

xbase_util-0.1.5/xbase_util/db/dao/CurrentConfigDao.py

@@ -0,0 +1,24 @@
+from xbase_util.db.bean.CurrentConfigBean import CurrentConfig
+
+
+class CurrentConfigDao:
+    def __init__(self,Session):
+        self.Session = Session
+
+    def set_current_config(self, id, desc):
+        with self.Session() as session:
+            try:
+                session.query(CurrentConfig).delete()
+                session.add(CurrentConfig(config_id=id, description=desc))
+                session.commit()
+            except Exception as e:
+                session.rollback()
+                print(f"Error: {e}")
+
+    def get_current_config(self):
+        with self.Session() as session:
+            try:
+                return session.query(CurrentConfig).first()
+            except Exception as e:
+                session.rollback()
+                print(f"Error: {e}")

xbase_util-0.1.5/xbase_util/db/dao/FlowDao.py

@@ -0,0 +1,59 @@
+from xbase_util.db.bean.FlowBean import FlowBean
+
+
+class FlowDao:
+    def __init__(self,Session):
+        self.Session = Session
+
+    def add_flow(self, description, step, flow_id=None):
+        with self.Session() as session:
+            try:
+                if flow_id is None:
+                    flow = FlowBean(description=description, step=step)
+                    session.add(flow)
+                else:
+                    flow = session.query(FlowBean).filter_by(id=flow_id).first()
+                    flow.description = description
+                    flow.step = step
+                session.commit()
+                return True
+            except Exception as e:
+                session.rollback()
+                print(e)
+                return False
+
+    def get_flow_list(self):
+        with self.Session() as session:
+            try:
+                flows = session.query(FlowBean).all()
+                return [{
+                    'id': item.id,
+                    'description': item.description,
+                    'step': item.step,
+                } for item in flows]
+            except Exception as e:
+                session.rollback()
+                print(e)
+                return []
+
+    def delete_by_id(self, id):
+        with self.Session() as session:
+            try:
+                flow = session.query(FlowBean).filter_by(id=id).first()
+                if flow:
+                    session.delete(flow)
+                    session.commit()
+                    return True
+            except Exception as e:
+                session.rollback()
+                print(e)
+                return False
+
+    def get_flow_by_id(self, id):
+        with self.Session() as session:
+            try:
+                return session.query(FlowBean).filter_by(id=id).first()
+            except Exception as e:
+                session.rollback()
+                print(e)
+                return None

xbase_util-0.1.5/xbase_util/db/dao/TaskTemplateDao.py

@@ -0,0 +1,57 @@
+from xbase_util.db.bean.TaskTemplateBean import TaskTemplateBean
+
+
+class TaskTemplateDao:
+    def __init__(self,Session):
+        self.Session = Session
+
+    def addTemplate(self, data):
+        with self.Session() as session:
+            try:
+                b = TaskTemplateBean()
+                b.config_id = data.config_id
+                b.flow_id = data.flow_id
+                b.description = data.description
+                b.is_scheduled = data.is_scheduled
+                b.scheduled_start_time = data.scheduled_start_time
+                b.scheduled_interval_minutes = data.scheduled_interval_minutes
+                b.scheduled_period_minutes = data.scheduled_period_minutes
+                session.add(b)
+                session.commit()
+            except Exception as e:
+                print(e)
+                session.rollback()
+
+    def changeTemplate(self, data):
+        with self.Session() as session:
+            try:
+                bean = session.query(TaskTemplateBean).first()
+                bean.config_id = data.config_id
+                bean.flow_id = data.flow_id
+                bean.description = data.description
+                session.commit()
+            except Exception as e:
+                print(e)
+                session.rollback()
+
+    def get_list(self):
+        with self.Session() as session:
+            try:
+                temp_list = session.query(TaskTemplateBean).all()
+                return temp_list
+            except Exception as e:
+                session.rollback()
+                print(e)
+                return []
+
+    def delete_template(self, id):
+        with self.Session() as session:
+            try:
+                bean = session.query(TaskTemplateBean).filter_by(id=id).first()
+                session.delete(bean)
+                session.commit()
+                return True
+            except Exception as e:
+                session.rollback()
+                print(e)
+                return False

xbase_util-0.1.5/xbase_util/db/initsqlite3.py

@@ -0,0 +1,18 @@
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from xbase_util.db.bean import DbBase
+from xbase_util.db.dao.ConfigDao import ConfigDao
+from xbase_util.db.dao.CurrentConfigDao import CurrentConfigDao
+from xbase_util.db.dao.FlowDao import FlowDao
+from xbase_util.db.dao.TaskTemplateDao import TaskTemplateDao
+
+
+def initSqlite3(path: str):
+    engine = create_engine(path, echo=False)
+    DbBase.metadata.create_all(engine)
+    Session = sessionmaker(bind=engine)
+    flowDao = FlowDao(Session)
+    configDao = ConfigDao(Session)
+    currentConfigDao = CurrentConfigDao(Session)
+    taskTemplateDao = TaskTemplateDao(Session)
+    return flowDao, configDao, currentConfigDao, taskTemplateDao

xbase_util-0.1.5/xbase_util/xbase_constant.py

@@ -0,0 +1,206 @@
+import os
+
+current_dir = os.path.dirname(__file__)
+parse_path = os.path.join(current_dir, '..', 'xbase_util_assets', 'arkimeparse.js')
+geo_path = os.path.join(current_dir, '..', 'xbase_util_assets', 'GeoLite2-City.mmdb')
+
+src_dst_header = ["src_host", "src_user_agent", "src_content_length", "src_connection", "src_content_type",
+                  "src_x_forwarded_for", "src_accept", "src_accept_encoding", "src_accept_language", "src_referer",
+                  "src_cookie", "src_origin", "src_x_requested_with", "src_sec_fetch_mode", "src_sec_fetch_site",
+                  "src_sec_fetch_dest", "src_content_encoding", "src_sec_ch_ua", "src_sec_ch_ua_mobile",
+                  "src_sec_ch_ua_platform", "src_cache_control", "src_upgrade_insecure_requests", "src_pragma",
+                  "src_x_real_ip", "src_soapaction", "src_x_forwarded_proto", "src_x_forwarded_host",
+                  "src_x_request_id",
+                  "src_x_forwarded_port", "src_x_forwarded_scheme", "src_x_original_forwarded_for", "src_x_scheme",
+                  "src_x_forwarded_scheme_diy", "src_authorization", "src_trace_id", "src_distinct_id", "src_url_path",
+                  "src_if_modified_since", "src_ecid_context", "src_token", "src_access_control_request_method",
+                  "src_access_token", "src_expect", "src_access_control_request_headers", "src_sec_fetch_user",
+                  "src_snk_location", "src_accept_charset", "src_date", "src_usercode", "src_logincode", "src_range",
+                  "src_checktime", "src_client_lang", "src_requestsource", "src_if_range", "src_if_none_match",
+                  "src_sw8",
+                  "src_sw8_correlation", "src_sw8_x", "src_x_sign", "src_x_timestamp", "src_charset",
+                  "src_x_custom_header", "src_sec_websocket_key", "src_sec_websocket_version", "src_upgrade",
+                  "src_sec_websocket_extensions", "src_xms_auth_token", "src_xms_cluster_id", "src_purpose",
+                  "src_x_ti_app_id", "src_x_ti_secret_code", "src_x_prototype_version", "src_systemcode",
+                  "src_access_control_request_private_network", "src_wxr", "src_client_version", "src_requestid",
+                  "src_request_module", "src_x_mule_encoding", "src_x_mule_session", "src_elite_tag", "src_dnt",
+                  "src_x_mule_endpoint", "src_x_mule_root_message_id", "src_x_mass_tappid",
+                  "src_x_elastic_product_origin",
+                  "src_priority", "src_server", "src_transfer_encoding", "src_xxl_job_access_token", "src_te",
+                  "src_access_control_allow_origin", "src_eset_spread_control", "src_x_splunk_digest",
+                  "src_x_splunk_lm_nonce", "src_x_splunk_lm_signature", "src_x_splunk_lm_timestamp", "src_avb_version",
+                  "src_set_cookie", "src_x_oracle_dms_ecid", "src_x_oracle_dms_rid",
+                  "src_x_prometheus_scrape_timeout_seconds", "src_client", "src_post", "src_ucosessionid", "src_user",
+                  "src_winuser", "src_showloading", "src_digest", "src_ocrimageflag", "src_x_content_type_options",
+                  "src_access_control_allow_methods", "src_access_control_allow_credentials", "src_expires",
+                  "src_keep_alive", "src_url_token", "src_postman_token", "src_x_xss_protection", "src_vary",
+                  "src_access_control_allow_headers", "src_signdata", "src_ua_cpu", "src_authorg", "src_yssip",
+                  "src_yssmac", "src_location", "src_content_language", "src_git_protocol", "src_x_forwarded_prefix",
+                  "src_invoke_type", "src_x_ua_compatible", "src_x_bd_traceid", "src_x_from_h3_trnet", "src_n",
+                  "src_nx_anti_csrf_token", "src_proxy_connection", "src_via", "src_x_nexus_ui", "src_pragma_type",
+                  "src_appkey", "src_area", "src_check", "src_clientname", "src_lang", "src_networktype", "src_nonce",
+                  "src_osversion", "src_screen", "src_st", "src_timestamp", "src_userid", "src_x_frame_options",
+                  "src_x_b3_parentspanid", "src_x_b3_sampled", "src_x_b3_spanid", "src_x_b3_traceid", "src_sec_purpose",
+                  "src_kbn_version", "src_access_control_expose_headers", "src_iszip", "src_from", "src_state",
+                  "src_access_control_max_age", "src_allow", "src_kbn_system_request", "src_tracelogcontext",
+                  "src_etag",
+                  "src_ms_cv", "src_x_application_context", "src_x_csrftoken", "src_tlogtraceid", "src_link_pwd_token",
+                  "src_preip", "src_preivkapp", "src_preivkhost", "src_tlogspanid", "src_accesstoken",
+                  "src_icy_metadata",
+                  "src_vivo_browser_text_zoom", "src_contents_client_adapter_id", "src_last_modified",
+                  "src_need_select_sub_url", "src_x_bd_quic", "src_x_bdboxapp_netengine", "src_x_playback_session_id",
+                  "src_x_turbonet_info", "src_cas_tgc", "src_imgids", "src_jsessionid", "src_accept_ranges",
+                  "src_am_traceid", "src_xms_backend_server", "src_ss_language", "src_sssessionid", "src_username",
+                  "src_content_md5", "src_x_acs_security_token", "src_x_log_apiversion", "src_x_log_bodyrawsize",
+                  "src_x_log_compresstype", "src_x_log_signaturemethod", "src_x_nod32_mode", "src_client_appname",
+                  "src_client_requesttoken", "src_client_requestts", "src_exconfiginfo", "src_long_pulling_timeout",
+                  "src_dispatch_header", "src_sign", "src_strict_transport_security", "src_ts", "src_bdpparallelload",
+                  "src_referrer_policy", "src_x_amz_request_id", "src_x_cm_service", "src_x_miorigin",
+                  "src_x_seafile_client_version", "src_amz_sdk_invocation_id", "src_amz_sdk_request",
+                  "src_amz_sdk_retry",
+                  "src_bussno", "src_busstype", "src_classcode", "src_comcode", "src_createtime", "src_createuser",
+                  "src_imgfilepath", "src_imgid", "src_imgtype", "src_s_cnection", "src_storagecondition",
+                  "src_x_eset_updateid", "src_x_powered_by", "src_allow_cross_domain_redirect",
+                  "src_amp_cache_transform",
+                  "src_x_csrf_token", "src_x_check_exist", "src_sec_gpc", "src_sf_ajax", "src_request_type",
+                  "src_x_envoy_decorator_operation", "src_x_envoy_upstream_service_time", "src_x_from_cdn",
+                  "src_x_download_options", "src_x_permitted_cross_domain_policies", "src_busscode", "src_companyno",
+                  "src_fcpos", "src_forwarded", "src_prod_sw8", "src_prod_sw8_correlation", "src_prod_sw8_x",
+                  "src_rewritepath", "src_x_elastic_product", "src_x_forwarded_server", "src_in_form_img", "src_x_pjax",
+                  "src_age", "src_channel", "src_client_ip", "src_content_transfer_encoding", "src_guid",
+                  "src_oldchannel",
+                  "src_product", "src_remote_addr", "src_seafile_repo_token", "src_starttag", "src_starttype",
+                  "src_traceid", "src_warded_for", "src_x_aspnet_version", "src_zcid", "src_agent_version",
+                  "src_alt_svc",
+                  "src_cf_ray", "src_chrome_proxy", "src_content_disposition", "src_content_range",
+                  "src_content_security_policy", "src_d_for", "src_grpc_accept_encoding", "src_grpc_timeout",
+                  "src_gslb_okhttp", "src_jenkins_crumb", "src_nel", "src_orderid", "src_p3p", "src_report_to",
+                  "src_route_data", "src_service_worker", "src_tap_app_conf_ver", "src_tap_gslb", "src_tc_anp",
+                  "src_tc_entsig", "src_tc_spanid", "src_tc_traceid", "src_www_authenticate", "src_x_aggregate_auth",
+                  "src_x_amz_storage_class", "src_x_cache_status", "src_x_ccc", "src_x_cdn_request_id", "src_x_cid",
+                  "src_x_clickhouse_format", "src_x_clickhouse_query_id", "src_x_clickhouse_server_display_name",
+                  "src_x_clickhouse_summary", "src_x_clickhouse_timezone", "src_x_gitlab_feature_category",
+                  "src_x_link_via", "src_x_ucbrowser_ua", "dst_date", "dst_content_type", "dst_content_length",
+                  "dst_transfer_encoding", "dst_connection", "dst_set_cookie", "dst_x_oracle_dms_ecid",
+                  "dst_x_oracle_dms_rid", "dst_content_language", "dst_cache_control", "dst_server", "dst_expires",
+                  "dst_pragma", "dst_access_control_allow_origin", "dst_keep_alive", "dst_vary", "dst_last_modified",
+                  "dst_access_control_allow_credentials", "dst_access_control_allow_methods", "dst_accept_ranges",
+                  "dst_access_control_allow_headers", "dst_x_content_type_options", "dst_etag",
+                  "dst_access_control_max_age", "dst_content_encoding", "dst_access_control_expose_headers",
+                  "dst_x_xss_protection", "dst_access_control_request_headers", "dst_x_frame_options", "dst_location",
+                  "dst_x_powered_by", "dst_x_application_context", "dst_p3p", "dst_x_ua_compatible",
+                  "dst_x_mule_encoding",
+                  "dst_x_mule_session", "dst_s_cnection", "dst_x_amz_request_id", "dst_content_disposition",
+                  "dst_x_amz_storage_class", "dst_content_range", "dst_pragma_type", "dst_allow",
+                  "dst_xms_backend_server",
+                  "dst_x_clickhouse_server_display_name", "dst_x_clickhouse_summary", "dst_x_clickhouse_format",
+                  "dst_x_clickhouse_query_id", "dst_x_clickhouse_timezone", "dst_referrer_policy",
+                  "dst_content_security_policy", "dst_content_id", "dst_sec_websocket_accept", "dst_upgrade",
+                  "dst_tlogtraceid", "dst_sec_websocket_extensions", "dst_x_elastic_product", "dst_via", "dst_x_cache",
+                  "dst_x_amz_cf_id", "dst_x_amz_cf_pop", "dst_rgwx_embedded_metadata_len", "dst_rgwx_mtime",
+                  "dst_rgwx_obj_pg_ver", "dst_rgwx_object_size", "dst_rgwx_source_zone_short_id",
+                  "dst_x_amz_version_id",
+                  "dst_x_request_id", "dst_x_envoy_decorator_operation", "dst_x_envoy_upstream_service_time", "dst_age",
+                  "dst_www_authenticate", "dst_strict_transport_security", "dst_terminationurl", "dst_x_splunk_digest",
+                  "dst_x_splunk_lm_nonce", "dst_x_splunk_lm_timestamp", "dst_x_download_options",
+                  "dst_x_permitted_cross_domain_policies", "dst_pragrma", "dst_content_transfer_encoding",
+                  "dst_x_runtime",
+                  "dst_x_arequestid", "dst_x_ausername", "dst_x_asessionid", "dst_x_aspnet_version",
+                  "dst_x_seraph_loginreason", "dst_accept_charset", "dst_n", "dst_error_code", "dst_error_msg",
+                  "dst_x_w_no", "dst_kbn_license_sig", "dst_kbn_name", "dst_cross_origin_opener_policy",
+                  "dst_xdomainrequestallowed", "dst_x_ratelimit_limit_vass_zuul_api_user_24",
+                  "dst_x_ratelimit_remaining_vass_zuul_api_use", "dst_x_ratelimit_remaining_vass_zuul_wx_port",
+                  "dst_x_ratelimit_reset_vass_zuul_api_user_24", "dst_audit_id", "dst_x_kubernetes_pf_flowschema_uid",
+                  "dst_x_kubernetes_pf_prioritylevel_uid", "dst_alt_svc", "dst_cf_ray", "dst_nel", "dst_report_to",
+                  "dst_x_ratelimit_remaining_vass_zuul_api_ord", "dst_x_protected_by", "dst_traceresponse",
+                  "dst_x_br_response", "dst_x_cache_status", "dst_x_ratelimit_limit_vass_zuul_api_order_1",
+                  "dst_x_ratelimit_reset_vass_zuul_api_order_1", "dst_grpc_metadata_accept_encoding",
+                  "dst_grpc_metadata_content_type", "dst_grpc_metadata_grpc_accept_encoding",
+                  "dst_x_ratelimit_limit_vass_zuul_wx_port_240", "dst_x_ratelimit_reset_vass_zuul_wx_port_240",
+                  "dst_permissions_policy", "dst_ctl_cache_status", "dst_progma", "dst_request_id", "dst_x_csrf_token",
+                  "dst_x_log_append_meta", "dst_x_log_requestid", "dst_x_log_time", "dst_hostname",
+                  "dst_x_networkmanager_status", "dst_browseruid", "dst_k_cache_status", "dst_kcs_via", "dst_x_ccc",
+                  "dst_x_cid", "dst_page_title", "dst_x_via", "dst_x_ws_request_id", "dst_enable_encrypted_library",
+                  "dst_expire", "dst_x_ratelimit_limit_vass_zuul_wx_port_39.",
+                  "dst_x_ratelimit_reset_vass_zuul_wx_port_39.", "dst_content_location", "dst_gsid", "dst_sc",
+                  "dst_x_response_timestrap", "dst_link", "dst_x_ratelimit_limit_vass_zuul_api_order_2",
+                  "dst_x_ratelimit_limit_vass_zuul_wx_port_10.", "dst_x_ratelimit_reset_vass_zuul_api_order_2",
+                  "dst_x_ratelimit_reset_vass_zuul_wx_port_10.", "dst_x_reqid", "dst_login", "dst_x_cache_lookup",
+                  "dst_x_cdn_request_id", "dst_x_link_via", "dst_x_nws_log_uuid",
+                  "dst_x_ratelimit_limit_vass_zuul_wx_port_111", "dst_x_ratelimit_reset_vass_zuul_wx_port_111",
+                  "dst_cache_contror", "dst_cdn_cache", "dst_cdn_cachedat", "dst_cdn_edgestorageid", "dst_cdn_proxyver",
+                  "dst_cdn_pullzone", "dst_cdn_requestcountrycode", "dst_cdn_requestid", "dst_cdn_requestpullcode",
+                  "dst_cdn_requestpullsuccess", "dst_cdn_status", "dst_cdn_uid", "dst_gitlab_ci_builds_polling",
+                  "dst_new_jwt", "dst_ohc_cache_hit", "dst_ohc_file_size", "dst_ohc_global_saved_time", "dst_x_hudson",
+                  "dst_x_instance_identity", "dst_x_jenkins", "dst_x_jenkins_session", "dst_dir_perm",
+                  "dst_fndfs_error",
+                  "dst_oid", "dst_x_errno", "dst_x_hudson_theme", "dst_x_oss_hash_crc64ecma", "dst_x_ser",
+                  "dst_content_md5", "dst_exception", "dst_exceptiontype", "dst_praga", "dst_x_oss_object_type",
+                  "dst_x_oss_request_id", "dst_x_oss_server_time", "dst_x_oss_storage_class",
+                  "dst_x_ratelimit_limit_vass_zuul_api_order_3", "dst_x_ratelimit_reset_vass_zuul_api_order_3"]
+dns_domain_list = ['ac.cnNEW', 'ah.cn', 'archiNEW', 'artHOT', 'asia', 'autoNEW', 'autosNEW', 'babyNEW', 'band',
+                   'beautyNEW',
+                   'beer', 'bioNEW', 'biz', 'bj.cn', 'blackNEW', 'blueNEW', 'bondNEW', 'cabNEW', 'cafeNEW', 'carNEW',
+                   'carsNEW',
+                   'cashNEW', 'cc', 'center', 'chat', 'cityNEW', 'clickNEW', 'cloud', 'clubHOT', 'cnHOT',
+                   'collegeNEW',
+                   'comHOT', 'com.cn', 'company', 'cool', 'cq.cn', 'cyouNEW', 'design', 'email', 'fanNEW', 'fans',
+                   'fashionNEW',
+                   'fit', 'fj.cn', 'fun', 'fund', 'fyiNEW', 'games', 'gd.cn', 'globalNEW', 'gold', 'gov.cn', 'greenNEW',
+                   'group',
+                   'gs.cn', 'guru', 'gx.cn', 'gz.cn', 'ha.cn', 'hairNEW', 'hb.cn', 'he.cn', 'hi.cn', 'hk.cn', 'hl.cn',
+                   'hn.cn',
+                   'homesNEW', 'host', 'icuHOT', 'info', 'ink', 'jl.cn', 'js.cn', 'jx.cn', 'kim', 'law', 'life', 'live',
+                   'ln.cn',
+                   'lottoNEW', 'love', 'ltdHOT', 'luxe', 'makeupNEW', 'market', 'mbaNEW', 'meNEW', 'mediaNEW', 'mo.cn',
+                   'mobi',
+                   'monsterNEW', 'motorcyclesNEW', 'net', 'net.cn', 'news', 'nm.cn', 'nx.cn', 'online', 'org.cn',
+                   'organicNEW',
+                   'pinkNEW', 'plus', 'pokerNEW', 'press', 'pro', 'promoNEW', 'protectionNEW', 'pub', 'pwNEW', 'qh.cn',
+                   'questNEW', 'red', 'ren', 'rentNEW', 'run', 'sc.cn', 'schoolNEW', 'sd.cn', 'securityNEW', 'sh.cn',
+                   'shopHOT',
+                   'shoppingNEW', 'show', 'site', 'skiNEW', 'skinNEW', 'sn.cn', 'socialNEW', 'space', 'storageNEW',
+                   'store',
+                   'studio', 'sx.cn', 'taxNEW', 'team', 'tech', 'technologyNEW', 'theatreNEW', 'ticketsNEW', 'tj.cn',
+                   'today',
+                   'topHOT', 'tvNEW', 'tw.cn', 'unoNEW', 'video', 'vinNEW', 'vip', 'voteNEW', 'votoNEW', 'wang',
+                   'website',
+                   'wiki', 'work', 'world', 'xin', 'xj.cn', 'xyz', 'xz.cn', 'yachtsNEW', 'yn.cn', 'yoga', 'zj.cn',
+                   'zone',
+                   '餐厅', '佛山', '公司', '广东', '集团', '企业NEW', '商标', '商城', '商店', '网店', '网络', '网址NEW',
+                   '我爱你', '游戏', '娱乐NEW', '在线', '招聘', '中国HOT', '中文网']
+statisticHeader = ['packet_size_mean', 'same_src_dst_size_mean', 'same_src_dst_size_var', 'packet_size_variance',
+                   'packet_len_total_count', 'packet_len_total_average',
+                   'packet_len_total_min', 'packet_len_total_max', 'packet_len_total_rate', 'packet_len_total_percent',
+                   'packet_len_0_19_count', 'packet_len_0_19_average', 'packet_len_0_19_min', 'packet_len_0_19_max',
+                   'packet_len_0_19_rate', 'packet_len_0_19_percent', 'packet_len_20_39_count',
+                   'packet_len_20_39_average', 'packet_len_20_39_min', 'packet_len_20_39_max', 'packet_len_20_39_rate',
+                   'packet_len_20_39_percent', 'packet_len_40_79_count', 'packet_len_40_79_average',
+                   'packet_len_40_79_min', 'packet_len_40_79_max', 'packet_len_40_79_rate', 'packet_len_40_79_percent',
+                   'packet_len_80_159_count', 'packet_len_80_159_average', 'packet_len_80_159_min',
+                   'packet_len_80_159_max', 'packet_len_80_159_rate', 'packet_len_80_159_percent',
+                   'packet_len_160_319_count', 'packet_len_160_319_average', 'packet_len_160_319_min',
+                   'packet_len_160_319_max', 'packet_len_160_319_rate', 'packet_len_160_319_percent',
+                   'packet_len_320_639_count', 'packet_len_320_639_average', 'packet_len_320_639_min',
+                   'packet_len_320_639_max', 'packet_len_320_639_rate', 'packet_len_320_639_percent',
+                   'packet_len_640_1279_count', 'packet_len_640_1279_average', 'packet_len_640_1279_min',
+                   'packet_len_640_1279_max', 'packet_len_640_1279_rate', 'packet_len_640_1279_percent',
+                   'packet_len_1280_2559_count', 'packet_len_1280_2559_average', 'packet_len_1280_2559_min',
+                   'packet_len_1280_2559_max', 'packet_len_1280_2559_rate', 'packet_len_1280_2559_percent',
+                   'packet_len_2560_5119_count', 'packet_len_2560_5119_average', 'packet_len_2560_5119_min',
+                   'packet_len_2560_5119_max', 'packet_len_2560_5119_rate', 'packet_len_2560_5119_percent',
+                   'packet_len_more_than_5120_count', 'packet_len_more_than_5120_average',
+                   'packet_len_more_than_5120_min', 'packet_len_more_than_5120_max', 'packet_len_more_than_5120_rate',
+                   'packet_len_more_than_5120_percent', 'all_req_packet_size_mean', 'all_req_packet_size_var',
+                   'all_res_packet_size_mean', 'all_res_packet_size_var', 'all_req_packet_time_period_mean',
+                   'all_res_packet_time_period_mean', 'all_req_packet_time_period_var',
+                   'all_res_packet_time_period_var', 'req_header_count_mean', 'req_header_count_var']
+features_key = [
+    'URI_FEATURES_EXTRA_contains_sql', 'URI_FEATURES_EXTRA_contains_xss', 'URI_FEATURES_EXTRA_contains_cmd',
+    'URI_FEATURES_EXTRA_contains_path', 'URI_FEATURES_EXTRA_contains_redirect',
+    'URI_FEATURES_EXTRA_contains_danger', 'URI_FEATURES_EXTRA_contains_suspicious_ext',
+    'URI_FEATURES_EXTRA_param_count', 'URI_FEATURES_EXTRA_path_depth', 'URI_FEATURES_EXTRA_param_length_avg',
+    'URI_FEATURES_EXTRA_param_length_max', 'UserAgent_is_attack', 'UserAgent_is_enterprise', 'UserAgent_browser',
+    'UserAgent_browser_version', 'UserAgent_os', 'UserAgent_os_version', 'UserAgent_device_type',
+    'UserAgent_platform', 'UserAgent_is_bot', 'UserAgent_language', 'UserAgent_special_char_count',
+    'UserAgent_is_unknown']

{xbase_util-0.1.3 → xbase_util-0.1.5}/xbase_util/xbase_util.py

@@ -4,9 +4,10 @@ from urllib.parse import urlparse, parse_qs
 
 import execjs
 import numpy as np
+import tldextract
 from scapy.layers.dns import DNS
 
-from xbase_util.xbase_constant import parse_path
+from xbase_util.xbase_constant import parse_path, dns_domain_list
 
 
 def parse_expression(expression):
@@ -379,3 +380,15 @@ def get_uri_filename_length(uri):
         extension = match.group(0)
         return len(extension)
     return 0
+
+
+def get_dns_domain_suffix(domain, dns_lock):
+    with dns_lock:
+        try:
+            for tmp_suffix in dns_domain_list:
+                if tmp_suffix in domain:
+                    return tmp_suffix
+            extracted = tldextract.extract(domain)
+            return extracted.suffix
+        except Exception as e:
+            return ""

{xbase_util-0.1.3 → xbase_util-0.1.5}/xbase_util.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase-util
-Version: 0.1.3
+Version: 0.1.5
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt

{xbase_util-0.1.3 → xbase_util-0.1.5}/xbase_util.egg-info/SOURCES.txt

@@ -14,4 +14,16 @@ xbase_util.egg-info/dependency_links.txt
 xbase_util.egg-info/not-zip-safe
 xbase_util.egg-info/top_level.txt
 xbase_util/../xbase_util_assets/GeoLite2-City.mmdb
-xbase_util/../xbase_util_assets/arkimeparse.js
+xbase_util/../xbase_util_assets/arkimeparse.js
+xbase_util/db/__init__.py
+xbase_util/db/initsqlite3.py
+xbase_util/db/bean/ConfigBean.py
+xbase_util/db/bean/CurrentConfigBean.py
+xbase_util/db/bean/FlowBean.py
+xbase_util/db/bean/TaskTemplateBean.py
+xbase_util/db/bean/__init__.py
+xbase_util/db/dao/ConfigDao.py
+xbase_util/db/dao/CurrentConfigDao.py
+xbase_util/db/dao/FlowDao.py
+xbase_util/db/dao/TaskTemplateDao.py
+xbase_util/db/dao/__init__.py

xbase_util-0.1.3/xbase_util/xbase_constant.py

@@ -1,5 +0,0 @@
-import os
-
-current_dir = os.path.dirname(__file__)
-parse_path = os.path.join(current_dir, '..', 'xbase_util_assets', 'arkimeparse.js')
-geo_path = os.path.join(current_dir, '..', 'xbase_util_assets', 'GeoLite2-City.mmdb')