xbase-util 0.1.2__tar.gz → 0.1.4__tar.gz

{xbase_util-0.1.2 → xbase_util-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase_util
-Version: 0.1.2
+Version: 0.1.4
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt

{xbase_util-0.1.2 → xbase_util-0.1.4}/setup.py

@@ -3,7 +3,7 @@ from distutils.core import setup
 from setuptools import find_packages
 
 setup(name="xbase_util",
-      version="0.1.2",
+      version="0.1.4",
       description="网络安全基础工具",
       long_description="包含提取，预测，训练的基础工具",
       author="xyt",

xbase_util-0.1.4/xbase_util/db/bean/ConfigBean.py

@@ -0,0 +1,103 @@
+from sqlalchemy import Column, Integer, String, TEXT, Boolean
+
+from xbase_util.db.bean import DbBase
+
+
+class ConfigBean(DbBase):
+    __tablename__ = 'configs'
+    id = Column(Integer, primary_key=True)
+    description = Column(TEXT)
+
+    label_all_true = Column(Boolean, nullable=False)
+    label_is_output_unmatch = Column(Boolean, nullable=False)
+    label_duration = Column(Integer, nullable=False)
+    splitNumber = Column(Integer, nullable=False)
+
+    mapping_le_path = Column(String, nullable=False)
+    pcap_per_subsection = Column(Integer, nullable=False)
+    pcap_process = Column(Integer, nullable=False)
+    pcap_thread_in_process = Column(Integer, nullable=False)
+    replace_source = Column(TEXT, nullable=False)  #用列表传
+    replace_destination = Column(String, nullable=False)
+    replace_mapping = Column(TEXT, nullable=False)  #用列表传
+    replace_save_to = Column(String, nullable=False)
+
+    session_all_true = Column(Boolean, nullable=False)
+    session_start_time = Column(String, nullable=False)
+    session_end_time = Column(String, nullable=False)
+    session_expression = Column(String)
+    session_alive = Column(String)
+    catalogue = Column(String)
+    session_size = Column(Integer)
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'description': self.description or "",
+
+            'label_all_true': self.label_all_true,
+            'label_is_output_unmatch': self.label_is_output_unmatch,
+            'label_duration': self.label_duration,
+            'splitNumber': self.splitNumber,
+
+            'mapping_le_path': self.mapping_le_path or "",
+
+            'pcap_per_subsection': self.pcap_per_subsection,
+            'pcap_process': self.pcap_process,
+            'pcap_thread_in_process': self.pcap_thread_in_process,
+
+            'replace_source': self.replace_source or "",
+            'replace_destination': self.replace_destination or "",
+            'replace_mapping': self.replace_mapping or "",
+            'replace_save_to': self.replace_save_to or "",
+
+            'session_all_true': self.session_all_true,
+            'session_start_time': self.session_start_time or "",
+            'session_end_time': self.session_end_time or "",
+            'session_expression': self.session_expression or "",
+            'session_alive': self.session_alive or "",
+            'session_size': self.session_size or "",
+            'catalogue': self.catalogue or "",
+        }
+
+    def to_session_dict(self):
+        return {
+            'id': self.id,
+            'session_all_true': self.session_all_true,
+            'session_start_time': self.session_start_time or "",
+            'session_end_time': self.session_end_time or "",
+            'session_expression': self.session_expression or "",
+            'session_alive': self.session_alive or "",
+            'session_size': self.session_size or "",
+        }
+
+    def to_pcap_dict(self):
+        return {
+            'id': self.id,
+            'pcap_per_subsection': self.pcap_per_subsection,
+            'pcap_process': self.pcap_process,
+            'pcap_thread_in_process': self.pcap_thread_in_process,
+        }
+
+    def to_label_dict(self):
+        return {
+            'id': self.id,
+            'label_all_true': self.label_all_true,
+            'label_is_output_unmatch': self.label_is_output_unmatch,
+            'label_duration': self.label_duration,
+        }
+
+    def to_mapping(self):
+        return {
+            'id': self.id,
+            'mapping_le_path': self.mapping_le_path or "",
+        }
+
+    def to_replace(self):
+        return {
+            'id': self.id,
+            'replace_source': self.replace_source or "",
+            'replace_destination': self.replace_destination or "",
+            'replace_mapping': self.replace_mapping or "",
+            'replace_save_to': self.replace_save_to or "",
+        }

xbase_util-0.1.4/xbase_util/db/bean/CurrentConfigBean.py

@@ -0,0 +1,10 @@
+from sqlalchemy import Column, Integer,String
+
+from xbase_util.db.bean import DbBase
+
+
+class CurrentConfig(DbBase):
+    __tablename__ = 'currentconfig'
+    id = Column(Integer, primary_key=True)
+    config_id = Column(Integer)
+    description = Column(String)

xbase_util-0.1.4/xbase_util/db/bean/FlowBean.py

@@ -0,0 +1,10 @@
+from sqlalchemy import Column, Integer, TEXT
+
+from xbase_util.db.bean import DbBase
+
+
+class FlowBean(DbBase):
+    __tablename__ = 'flows'
+    id = Column(Integer, primary_key=True)
+    description = Column(TEXT)
+    step = Column(TEXT)

xbase_util-0.1.4/xbase_util/db/bean/TaskTemplateBean.py

@@ -0,0 +1,28 @@
+from sqlalchemy import Column, Integer, String, Boolean
+
+from xbase_util.db.bean import DbBase
+
+
+class TaskTemplateBean(DbBase):
+    __tablename__ = 'tasktemplatebean'
+    id = Column(Integer, primary_key=True)
+
+    config_id = Column(String)
+    flow_id = Column(String)
+    description = Column(String)
+
+    is_scheduled = Column(Boolean, default=False)  # 是否为定时任务
+    scheduled_start_time = Column(String, nullable=True)  # 定时任务的开始时间
+    scheduled_interval_minutes = Column(Integer, nullable=True)  # 定时任务的执行间隔（以分钟为单位）
+    scheduled_period_minutes = Column(Integer, nullable=True)  # 要获取的时间段（以分钟为单位）
+
+    def to_dict(self):
+        return {
+            "id": self.id,
+            "config_id": self.config_id,
+            "flow_id": self.flow_id,
+            "description": self.description,
+            "is_scheduled": self.is_scheduled,
+            "start_time": self.scheduled_start_time,
+            "interval_minutes": self.scheduled_interval_minutes,
+        }

xbase_util-0.1.4/xbase_util/db/bean/__init__.py

@@ -0,0 +1,3 @@
+from sqlalchemy.orm import declarative_base
+
+DbBase = declarative_base()

xbase_util-0.1.4/xbase_util/db/dao/ConfigDao.py

@@ -0,0 +1,172 @@
+import traceback
+
+from xbase_util.db.bean.ConfigBean import ConfigBean
+
+
+class ConfigDao:
+    def __init__(self,Session):
+        self.Session = Session
+
+    def add(self, id, description, label_all_true, label_is_output_unmatch,
+            label_duration,
+            mapping_le_path, pcap_per_subsection, pcap_process, pcap_thread_in_process,
+            replace_source, replace_destination, replace_mapping, replace_save_to, session_all_true, session_start_time,
+            session_end_time, session_expression, session_alive, session_size, splitNumber,catalogue
+            ):
+        with self.Session() as session:
+            try:
+                if id is None:
+                    bean = ConfigBean(
+                        description=description,
+                        label_all_true=label_all_true,
+                        label_is_output_unmatch=label_is_output_unmatch,
+                        label_duration=label_duration,
+                        mapping_le_path=mapping_le_path,
+                        pcap_per_subsection=pcap_per_subsection,
+                        pcap_process=pcap_process,
+                        pcap_thread_in_process=pcap_thread_in_process,
+                        replace_source=replace_source,
+                        replace_destination=replace_destination,
+                        replace_mapping=replace_mapping,
+                        replace_save_to=replace_save_to,
+                        session_all_true=session_all_true,
+                        session_start_time=session_start_time,
+                        session_end_time=session_end_time,
+                        session_expression=session_expression,
+                        session_alive=session_alive,
+                        session_size=session_size,
+                        splitNumber=splitNumber,
+                        catalogue=catalogue
+                    )
+                    session.add(bean)
+                    session.commit()
+                    return True
+                else:
+                    config = session.query(ConfigBean).filter(ConfigBean.id == id).first()
+                    if config is None:
+                        return False
+                    config.description = description
+                    config.label_all_true = label_all_true
+                    config.label_is_output_unmatch = label_is_output_unmatch
+                    config.label_duration = label_duration
+                    config.mapping_le_path = mapping_le_path
+                    config.pcap_per_subsection = pcap_per_subsection
+                    config.pcap_process = pcap_process
+                    config.pcap_thread_in_process = pcap_thread_in_process
+                    config.replace_source = replace_source
+                    config.replace_destination = replace_destination
+                    config.replace_mapping = replace_mapping
+                    config.replace_save_to = replace_save_to
+                    config.session_all_true = session_all_true
+                    config.session_start_time = session_start_time
+                    config.session_end_time = session_end_time
+                    config.session_expression = session_expression
+                    config.session_alive = session_alive
+                    config.session_size = session_size
+                    config.splitNumber = splitNumber
+                    config.catalogue=catalogue
+                    session.commit()
+            except Exception as e:
+                session.rollback()
+                traceback.print_exc()
+                print(e)
+
+    def get_config_file_list(self):
+        with self.Session() as session:
+            try:
+                config_list = session.query(ConfigBean).all()
+                return [d.to_dict() for d in config_list]
+            except Exception as e:
+                session.rollback()
+                print(e)
+
+    def remove_by_id(self, id):
+        with self.Session() as session:
+            try:
+                session.query(ConfigBean).filter(ConfigBean.id == id).delete()
+                session.commit()
+            except Exception as e:
+                session.rollback()
+                print(e)
+
+    def get_config_by_id(self, id):
+        with self.Session() as session:
+            try:
+                return session.query(ConfigBean).filter(ConfigBean.id == id).first()
+            except Exception as e:
+                session.rollback()
+                print(e)
+
+    def set_config_session_by_id(self, id, session_all_true, session_start_time, session_end_time, session_expression,
+                                 session_alive, session_size):
+        with self.Session() as session:
+            try:
+                config = session.query(ConfigBean).filter(ConfigBean.id == id).first()
+                config.session_all_true = session_all_true
+                config.session_start_time = session_start_time
+                config.session_end_time = session_end_time
+                config.session_expression = session_expression
+                config.session_alive = session_alive
+                config.session_size = session_size
+                session.commit()
+            except Exception as e:
+                session.rollback()
+                print(e)
+
+    def set_config_pcap_by_id(self, id,
+                              pcap_per_subsection,
+                              pcap_process,
+                              pcap_thread_in_process):
+        with self.Session() as session:
+            try:
+                config = session.query(ConfigBean).filter(ConfigBean.id == id).first()
+                config.pcap_per_subsection = pcap_per_subsection
+                config.pcap_process = pcap_process
+                config.pcap_thread_in_process = pcap_thread_in_process
+                session.commit()
+            except Exception as e:
+                session.rollback()
+                print(e)
+
+    def set_config_label_by_id(self, id,
+                               label_all_true,
+                               label_is_output_unmatch,
+                               label_duration):
+        with self.Session() as session:
+            try:
+                config = session.query(ConfigBean).filter(ConfigBean.id == id).first()
+                config.label_all_true = label_all_true
+                config.label_is_output_unmatch = label_is_output_unmatch
+                config.label_duration = label_duration
+                session.commit()
+            except Exception as e:
+                session.rollback()
+                print(e)
+
+    def set_config_mapping_by_id(self, id, mapping_le_path):
+        with self.Session() as session:
+            try:
+                config = session.query(ConfigBean).filter(ConfigBean.id == id).first()
+                config.mapping_le_path = mapping_le_path
+                session.commit()
+            except Exception as e:
+                session.rollback()
+                print(e)
+
+    def set_config_replace_by_id(self, id, replace_source, replace_destination, replace_mapping, replace_save_to):
+        with self.Session() as session:
+            try:
+                config = session.query(ConfigBean).filter(ConfigBean.id == id).first()
+                config.replace_source = replace_source
+                config.replace_destination = replace_destination
+                config.replace_mapping = replace_mapping
+                config.replace_save_to = replace_save_to
+                session.commit()
+            except Exception as e:
+                session.rollback()
+                print(e)
+# 修改脚本带redis获取状态
+# 提异常，正常dns和异常都要，要新的黑白样本
+# app
+# capture
+# 日报规范问题 [做了什么][完成的进度][遇到的问题][问题研究的进度和方案]

xbase_util-0.1.4/xbase_util/db/dao/CurrentConfigDao.py

@@ -0,0 +1,24 @@
+from xbase_util.db.bean.CurrentConfigBean import CurrentConfig
+
+
+class CurrentConfigDao:
+    def __init__(self,Session):
+        self.Session = Session
+
+    def set_current_config(self, id, desc):
+        with self.Session() as session:
+            try:
+                session.query(CurrentConfig).delete()
+                session.add(CurrentConfig(config_id=id, description=desc))
+                session.commit()
+            except Exception as e:
+                session.rollback()
+                print(f"Error: {e}")
+
+    def get_current_config(self):
+        with self.Session() as session:
+            try:
+                return session.query(CurrentConfig).first()
+            except Exception as e:
+                session.rollback()
+                print(f"Error: {e}")

xbase_util-0.1.4/xbase_util/db/dao/FlowDao.py

@@ -0,0 +1,59 @@
+from xbase_util.db.bean.FlowBean import FlowBean
+
+
+class FlowDao:
+    def __init__(self,Session):
+        self.Session = Session
+
+    def add_flow(self, description, step, flow_id=None):
+        with self.Session() as session:
+            try:
+                if flow_id is None:
+                    flow = FlowBean(description=description, step=step)
+                    session.add(flow)
+                else:
+                    flow = session.query(FlowBean).filter_by(id=flow_id).first()
+                    flow.description = description
+                    flow.step = step
+                session.commit()
+                return True
+            except Exception as e:
+                session.rollback()
+                print(e)
+                return False
+
+    def get_flow_list(self):
+        with self.Session() as session:
+            try:
+                flows = session.query(FlowBean).all()
+                return [{
+                    'id': item.id,
+                    'description': item.description,
+                    'step': item.step,
+                } for item in flows]
+            except Exception as e:
+                session.rollback()
+                print(e)
+                return []
+
+    def delete_by_id(self, id):
+        with self.Session() as session:
+            try:
+                flow = session.query(FlowBean).filter_by(id=id).first()
+                if flow:
+                    session.delete(flow)
+                    session.commit()
+                    return True
+            except Exception as e:
+                session.rollback()
+                print(e)
+                return False
+
+    def get_flow_by_id(self, id):
+        with self.Session() as session:
+            try:
+                return session.query(FlowBean).filter_by(id=id).first()
+            except Exception as e:
+                session.rollback()
+                print(e)
+                return None

xbase_util-0.1.4/xbase_util/db/dao/TaskTemplateDao.py

@@ -0,0 +1,57 @@
+from xbase_util.db.bean.TaskTemplateBean import TaskTemplateBean
+
+
+class TaskTemplateDao:
+    def __init__(self,Session):
+        self.Session = Session
+
+    def addTemplate(self, data):
+        with self.Session() as session:
+            try:
+                b = TaskTemplateBean()
+                b.config_id = data.config_id
+                b.flow_id = data.flow_id
+                b.description = data.description
+                b.is_scheduled = data.is_scheduled
+                b.scheduled_start_time = data.scheduled_start_time
+                b.scheduled_interval_minutes = data.scheduled_interval_minutes
+                b.scheduled_period_minutes = data.scheduled_period_minutes
+                session.add(b)
+                session.commit()
+            except Exception as e:
+                print(e)
+                session.rollback()
+
+    def changeTemplate(self, data):
+        with self.Session() as session:
+            try:
+                bean = session.query(TaskTemplateBean).first()
+                bean.config_id = data.config_id
+                bean.flow_id = data.flow_id
+                bean.description = data.description
+                session.commit()
+            except Exception as e:
+                print(e)
+                session.rollback()
+
+    def get_list(self):
+        with self.Session() as session:
+            try:
+                temp_list = session.query(TaskTemplateBean).all()
+                return temp_list
+            except Exception as e:
+                session.rollback()
+                print(e)
+                return []
+
+    def delete_template(self, id):
+        with self.Session() as session:
+            try:
+                bean = session.query(TaskTemplateBean).filter_by(id=id).first()
+                session.delete(bean)
+                session.commit()
+                return True
+            except Exception as e:
+                session.rollback()
+                print(e)
+                return False

xbase_util-0.1.4/xbase_util/db/initsqlite3.py

@@ -0,0 +1,18 @@
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from xbase_util.db.bean import DbBase
+from xbase_util.db.dao.ConfigDao import ConfigDao
+from xbase_util.db.dao.CurrentConfigDao import CurrentConfigDao
+from xbase_util.db.dao.FlowDao import FlowDao
+from xbase_util.db.dao.TaskTemplateDao import TaskTemplateDao
+
+
+def initSqlite3(path: str):
+    engine = create_engine(path, echo=False)
+    DbBase.metadata.create_all(engine)
+    Session = sessionmaker(bind=engine)
+    flowDao = FlowDao(Session)
+    configDao = ConfigDao(Session)
+    currentConfigDao = CurrentConfigDao(Session)
+    taskTemplateDao = TaskTemplateDao(Session)
+    return flowDao, configDao, currentConfigDao, taskTemplateDao

{xbase_util-0.1.2 → xbase_util-0.1.4}/xbase_util/xbase_util.py

@@ -230,7 +230,7 @@ def get_dns_domain(packets):
     return domain_name
 
 
-def extract_session_fields(cls, origin_list, geoUtil):
+def extract_session_fields(origin_list, geoUtil):
     res = []
     for item in origin_list:
         _source = item.get("_source", {})
@@ -243,8 +243,8 @@ def extract_session_fields(cls, origin_list, geoUtil):
         uri = http.get('uri', [])
         uri_length = [len(u) for u in uri]
         uri_depth = [get_uri_depth(u) for u in uri]
-        uri_filename_length = [cls.get_uri_filename_length(u) for u in uri]
-        uri_params = [cls.get_url_param_count(u) for u in uri]
+        uri_filename_length = [get_uri_filename_length(u) for u in uri]
+        uri_params = [get_url_param_count(u) for u in uri]
         res.append(geoUtil.get_geo_by_ip({
             "id": item["_id"],
             "node": _source.get("node", ""),

{xbase_util-0.1.2 → xbase_util-0.1.4}/xbase_util.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xbase-util
-Version: 0.1.2
+Version: 0.1.4
 Summary: 网络安全基础工具
 Home-page: https://gitee.com/jimonik/xbase_util.git
 Author: xyt

{xbase_util-0.1.2 → xbase_util-0.1.4}/xbase_util.egg-info/SOURCES.txt

@@ -14,4 +14,16 @@ xbase_util.egg-info/dependency_links.txt
 xbase_util.egg-info/not-zip-safe
 xbase_util.egg-info/top_level.txt
 xbase_util/../xbase_util_assets/GeoLite2-City.mmdb
-xbase_util/../xbase_util_assets/arkimeparse.js
+xbase_util/../xbase_util_assets/arkimeparse.js
+xbase_util/db/__init__.py
+xbase_util/db/initsqlite3.py
+xbase_util/db/bean/ConfigBean.py
+xbase_util/db/bean/CurrentConfigBean.py
+xbase_util/db/bean/FlowBean.py
+xbase_util/db/bean/TaskTemplateBean.py
+xbase_util/db/bean/__init__.py
+xbase_util/db/dao/ConfigDao.py
+xbase_util/db/dao/CurrentConfigDao.py
+xbase_util/db/dao/FlowDao.py
+xbase_util/db/dao/TaskTemplateDao.py
+xbase_util/db/dao/__init__.py