mmisp-lib 0.7.5__tar.gz → 0.8.0__tar.gz

{mmisp_lib-0.7.5 → mmisp_lib-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: mmisp-lib
-Version: 0.7.5
+Version: 0.8.0
 Requires-Python: >=3.11.0
 Description-Content-Type: text/markdown
 License-File: LICENSE

{mmisp_lib-0.7.5 → mmisp_lib-0.8.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mmisp-lib"
-version = "0.7.5"
+version = "0.8.0"
 description = ""
 authors = []
 readme = "README.md"

{mmisp_lib-0.7.5 → mmisp_lib-0.8.0}/src/mmisp/db/config.py

@@ -22,6 +22,7 @@ class DatabaseConfig:
     DEBUG: bool
     RETRY_SLEEP: int
     MAX_RETRIES: int
+    DB_LOGLEVEL: str | None = None
 
 
 load_dotenv(getenv("ENV_FILE", ".env"))
@@ -32,8 +33,12 @@ config: DatabaseConfig = DatabaseConfig(
     DEBUG=bool(getenv("DEBUG", False)),
     RETRY_SLEEP=int(getenv("DB_RETRY", 5)),
     MAX_RETRIES=int(getenv("DB_MAX_RETRIES", 100)),
+    DB_LOGLEVEL=getenv("DB_LOGLEVEL", None),
 )
 sqlalchemy_logger = logging.getLogger("sqlalchemy.engine")
-sqlalchemy_logger.setLevel(logging.INFO)
+sqlalchemy_logger.setLevel(logging.WARNING)
 if config.DEBUG:
-    sqlalchemy_logger.setLevel(logging.DEBUG)
+    sqlalchemy_logger.setLevel(logging.INFO)
+
+if config.DB_LOGLEVEL is not None:
+    sqlalchemy_logger.setLevel(config.DB_LOGLEVEL)

mmisp_lib-0.8.0/src/mmisp/db/lib.py

@@ -0,0 +1,17 @@
+import logging
+
+from sqlalchemy import func
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.sql import Select
+
+logger = logging.getLogger("mmisp")
+
+
+async def get_count_from_select(db: AsyncSession, query: Select) -> int:
+    logger.debug("Get count from query: %s", query)
+    count_query = (
+        query.with_only_columns(func.count(), maintain_column_froms=True).order_by(None).limit(None).offset(None)
+    )
+    logger.debug("Resulted count query: %s", count_query)
+    result = await db.execute(count_query)
+    return result.scalar()

{mmisp_lib-0.7.5 → mmisp_lib-0.8.0}/src/mmisp/db/models/log.py

@@ -1,33 +1,68 @@
-from datetime import datetime
-
-from sqlalchemy import DateTime, Integer, String, Text
-
-from mmisp.db.mypy import Mapped, mapped_column
-
-from ..database import Base
-
-
-class Log(Base):
-    """
-    A python class representation of the database model for logs in MISP.
-
-    Further explanation for some of the central attributes of the database model:
-    - Action: Describes the action that was logged, e.g. a login or workflow execution
-    - Change: A string-representation of the changes made to the logged object or of
-              central information about the logged object.
-    """
-
-    __tablename__ = "logs"
-
-    id: Mapped[int] = mapped_column(Integer, primary_key=True, nullable=False)
-    title: Mapped[str] = mapped_column(Text, nullable=True)
-    created: Mapped[datetime] = mapped_column(DateTime, nullable=False, default=datetime.utcnow)
-    model: Mapped[str] = mapped_column(String(80), nullable=False)
-    model_id: Mapped[int] = mapped_column(Integer, nullable=False)
-    action: Mapped[str] = mapped_column(String(20), nullable=False)
-    user_id: Mapped[int] = mapped_column(Integer, nullable=False)
-    change: Mapped[str] = mapped_column(Text, nullable=True)
-    email: Mapped[str] = mapped_column(String(255), nullable=False)
-    org: Mapped[str] = mapped_column(String(255), nullable=False)
-    description: Mapped[str] = mapped_column(Text, nullable=True)
-    ip: Mapped[str] = mapped_column(String(45), nullable=True)
+from datetime import datetime
+from typing import Self
+
+from sqlalchemy import DateTime, Integer, String, Text
+
+from mmisp.db.mypy import Mapped, mapped_column
+
+from ..database import Base
+
+
+class Log(Base):
+    """
+    A python class representation of the database model for logs in MISP.
+
+    Further explanation for some of the central attributes of the database model:
+    - Action: Describes the action that was logged, e.g. a login or workflow execution
+    - Change: A string-representation of the changes made to the logged object or of
+              central information about the logged object.
+    """
+
+    __tablename__ = "logs"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, nullable=False)
+    title: Mapped[str] = mapped_column(Text, nullable=True)
+    created: Mapped[datetime] = mapped_column(DateTime, nullable=False, default=datetime.utcnow)
+    model: Mapped[str] = mapped_column(String(80), nullable=False)
+    model_id: Mapped[int] = mapped_column(Integer, nullable=False)
+    action: Mapped[str] = mapped_column(String(20), nullable=False)
+    user_id: Mapped[int] = mapped_column(Integer, nullable=False)
+    change: Mapped[str] = mapped_column(Text, nullable=True)
+    email: Mapped[str] = mapped_column(String(255), nullable=False)
+    org: Mapped[str] = mapped_column(String(255), nullable=False)
+    description: Mapped[str] = mapped_column(Text, nullable=True)
+    ip: Mapped[str] = mapped_column(String(45), nullable=True)
+
+    def __init__(
+        self: Self,
+        *,
+        model: str,
+        model_id: int,
+        action: str,
+        user_id: int,
+        email: str,
+        org: str,
+        title: str | None = None,
+        change: str | None = None,
+        description: str | None = None,
+        ip: str | None = None,
+        created: datetime | None = None,
+        **kwargs,
+    ) -> None:
+        if created is None:
+            created = datetime.now()
+        if isinstance(created, float):
+            created = datetime.fromtimestamp(created)
+        super().__init__(
+            title=title,
+            created=created,
+            model=model,
+            model_id=model_id,
+            action=action,
+            user_id=user_id,
+            change=change,
+            email=email,
+            org=org,
+            description=description,
+            ip=ip,
+        )

{mmisp_lib-0.7.5 → mmisp_lib-0.8.0}/src/mmisp/db/models/workflow.py

@@ -1,3 +1,6 @@
+import logging
+from functools import cached_property
+from typing import Self
 from uuid import uuid4 as _uuid4
 
 from sqlalchemy import Boolean, Integer, String
@@ -7,6 +10,7 @@ from mmisp.db.mypy import Mapped, mapped_column
 from ...workflows.graph import WorkflowGraph
 from ...workflows.legacy import JSONGraphType
 from ..database import Base
+from .log import Log
 
 
 def uuid() -> str:
@@ -36,3 +40,30 @@ class Workflow(Base):
     trigger_id: Mapped[str] = mapped_column(String(191), nullable=False, index=True)
     debug_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=0)
     data: Mapped[WorkflowGraph] = mapped_column(JSONGraphType, nullable=False, default=0)
+
+    def __init__(self: Self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+
+    @cached_property
+    def _logger(self: Self) -> logging.LoggerAdapter:
+        _logger = logging.LoggerAdapter(
+            logging.getLogger(f"mmisp.workflows.{self.name}"),
+            extra=dict(
+                dbmodel=Log,
+                model="Workflow",
+                model_id=self.id,
+                action="execute_workflow",
+                user_id=0,
+                email="SYSTEM",
+                org="SYSTEM",
+                description="",
+                change="",
+                ip="",
+            ),
+        )
+        if self.debug_enabled:
+            _logger.setLevel(logging.DEBUG)
+        return _logger
+
+    def get_logger(self: Self) -> logging.LoggerAdapter:
+        return self._logger

{mmisp_lib-0.7.5 → mmisp_lib-0.8.0}/src/mmisp/lib/logger.py

@@ -3,8 +3,10 @@ import logging
 from contextvars import ContextVar
 from typing import Self, TypeVar
 
+from sqlalchemy.ext.asyncio import AsyncSession
+
 request_log: ContextVar[list] = ContextVar("request_log")
-# db_log: ContextVar[list] = ContextVar("db_log")
+db_log: ContextVar[list] = ContextVar("db_log")
 
 
 T = TypeVar("T")
@@ -56,20 +58,47 @@ class InMemoryContextHandler(logging.Handler):
         request_log.get([]).append(self.format(record))
 
 
+class InMemoryDBLogContextHandler(logging.Handler):
+    def emit(self: Self, record: logging.LogRecord) -> None:
+        if hasattr(record, "dbmodel"):
+            dbmodel_class = getattr(record, "dbmodel")
+            instance = dbmodel_class(
+                title=self.format(record),
+                **record.__dict__,
+            )
+            db_log.get([]).append(instance)
+
+
 def print_request_log() -> None:
     print(*request_log.get([]), sep="\n")
 
 
+async def save_db_log(db: AsyncSession) -> None:
+    db.add_all(db_log.get([]))
+    await db.flush()
+
+
 def reset_request_log() -> None:
     request_log.set([])
 
 
+def reset_db_log() -> None:
+    db_log.set([])
+
+
 # Set up the logger
 logger = logging.getLogger("mmisp")
 in_memory_handler = InMemoryContextHandler()
+in_memory_db_log_handler = InMemoryDBLogContextHandler()
 log_formatter = logging.Formatter("%(asctime)s %(levelname)s: %(message)s")
+dblog_formatter = logging.Formatter("%(message)s")
 in_memory_handler.setFormatter(log_formatter)
+in_memory_db_log_handler.setFormatter(dblog_formatter)
 logger.addHandler(in_memory_handler)
+logger.addHandler(in_memory_db_log_handler)
+
+# adapter
+
 
 sqlalchemy_logger = logging.getLogger("sqlalchemy.engine")
 sqlalchemy_logger.addHandler(in_memory_handler)

{mmisp_lib-0.7.5 → mmisp_lib-0.8.0}/src/mmisp/workflows/execution.py

@@ -15,7 +15,6 @@ from ..db.models.organisation import Organisation
 from ..db.models.role import Role
 from ..db.models.user import User
 from ..db.models.workflow import Workflow
-from ..lib.logging import ApplicationLogger
 from .graph import Module, Trigger, VerbatimWorkflowInput
 from .input import WorkflowInput
 from .modules import ModuleLogic
@@ -45,7 +44,6 @@ async def walk_nodes(
     input: WorkflowInput,
     current_node: Module,
     workflow: Workflow,
-    logger: ApplicationLogger,
     db: AsyncSession,
     jinja2_engine: Environment,
 ) -> Tuple[bool, List[str]]:
@@ -59,8 +57,6 @@ async def walk_nodes(
         input:          Workflow payload for `current_node`.
         current_node:   Node to resume execution with.
         workflow:       Workflow entity. Used for logging.
-        logger:         Application logger to write debug messages and errors
-            from the execution.
         db:             Database session.
         jinja2_engine:  Instantiated templating engine to substitute placeholders
             in module configuration with values from the payload.
@@ -85,13 +81,12 @@ async def walk_nodes(
                         current_config[config_key] = [_render(v) for v in value]
         result, next_node = await current_node.exec(input, db)
     except Exception as e:
-        logger.log_workflow_execution_error(workflow, f"Error while executing module {current_node.id}. Error: {e}")
+        workflow.get_logger().error(f"Error while executing module {current_node.id}. Error: {e}")
         return False, []
 
     success_type = "partial-success" if not result and not isinstance(current_node, ModuleLogic) else "success"
 
-    logger.log_workflow_debug_message(
-        workflow,
+    workflow.get_logger().debug(
         f"Executed node `{current_node.id}`\n"
         + f"Node `{current_node.id}` from Workflow `{workflow.name}` ({workflow.id}) executed "
         + f"successfully with status: {success_type}",
@@ -105,7 +100,7 @@ async def walk_nodes(
 
     # At this stage we don't do any cycle detection, but assume that only
     # valid graphs w/o cycles in it were saved by the API.
-    return await walk_nodes(input, next_node, workflow, logger, db, jinja2_engine)
+    return await walk_nodes(input, next_node, workflow, db, jinja2_engine)
 
 
 async def create_virtual_root_user(db: AsyncSession) -> User:
@@ -141,7 +136,7 @@ async def workflow_by_trigger_id(trigger: str, db: AsyncSession) -> Workflow | N
 
 
 async def execute_workflow(
-    workflow: Workflow, user: User, input: VerbatimWorkflowInput, db: AsyncSession, logger: ApplicationLogger
+    workflow: Workflow, user: User, input: VerbatimWorkflowInput, db: AsyncSession
 ) -> Tuple[bool, List[str]]:
     """
     Provides the functionality for executing a workflow, which consists of traversing
@@ -169,7 +164,6 @@ async def execute_workflow(
         workflow: The Graph representation of the workflow to be executed.
         input:    Initial payload for the workflow.
         db:       SQLAlchemy session.
-        logger:   Application logger to notify about errors or (if debug is enabled) execution steps.
     """
 
     if not workflow.enabled:
@@ -188,8 +182,7 @@ async def execute_workflow(
 
     if len(unsupported_modules_id) != 0:
         unsupported_modules_str = ", ".join(unsupported_modules_id)
-        logger.log_workflow_execution_error(
-            workflow,
+        workflow.get_logger().error(
             "Workflow was not executed, because it contained unsupported modules with the following IDs: "
             + unsupported_modules_str,
         )
@@ -198,9 +191,7 @@ async def execute_workflow(
             + unsupported_modules_str
         ]
 
-    logger.log_workflow_debug_message(
-        workflow, f"Started executing workflow for trigger `{trigger.name}` ({workflow.id})"
-    )
+    workflow.get_logger().debug(f"Started executing workflow for trigger `{trigger.name}` ({workflow.id})")
 
     next_step = next(iter(trigger.outputs.values()), None)
     # Nothing to do.
@@ -211,7 +202,7 @@ async def execute_workflow(
         roaming_data = await trigger.normalize_data(db, input)
     except Exception as e:
         await db.rollback()
-        logger.log_workflow_execution_error(workflow, f"Error while normalizing data for trigger. Error: \n{e}")
+        workflow.get_logger().error(f"Error while normalizing data for trigger. Error: \n{e}")
         return False, [f"Internal error: {e}"]
 
     input = WorkflowInput(
@@ -222,17 +213,15 @@ async def execute_workflow(
 
     await _increase_workflow_execution_count(db, workflow.id)
 
-    result = await walk_nodes(
-        input, _as_module(next_step[0][1]), workflow, logger, db, Environment(loader=BaseLoader())
-    )
+    result = await walk_nodes(input, _as_module(next_step[0][1]), workflow, db, Environment(loader=BaseLoader()))
 
     if result[0]:
         outcome = "success"
     else:
         outcome = "blocked" if trigger.blocking else "failure"
 
-    logger.log_workflow_debug_message(
-        workflow, f"Finished executing workflow for trigger `{trigger.name}` ({workflow.id}). Outcome: {outcome}"
+    workflow.get_logger().debug(
+        f"Finished executing workflow for trigger `{trigger.name}` ({workflow.id}). Outcome: {outcome}"
     )
 
     if not result[0]:

{mmisp_lib-0.7.5 → mmisp_lib-0.8.0}/src/mmisp_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: mmisp-lib
-Version: 0.7.5
+Version: 0.8.0
 Requires-Python: >=3.11.0
 Description-Content-Type: text/markdown
 License-File: LICENSE

{mmisp_lib-0.7.5 → mmisp_lib-0.8.0}/src/mmisp_lib.egg-info/SOURCES.txt

@@ -44,6 +44,7 @@ src/mmisp/db/additional_properties.py
 src/mmisp/db/all_models.py
 src/mmisp/db/config.py
 src/mmisp/db/database.py
+src/mmisp/db/lib.py
 src/mmisp/db/mixins.py
 src/mmisp/db/mypy.py
 src/mmisp/db/print_changes.py
@@ -87,7 +88,6 @@ src/mmisp/lib/fallbacks.py
 src/mmisp/lib/galaxies.py
 src/mmisp/lib/galaxy_clusters.py
 src/mmisp/lib/logger.py
-src/mmisp/lib/logging.py
 src/mmisp/lib/permissions.py
 src/mmisp/lib/py.typed
 src/mmisp/lib/serialisation_helper.py

mmisp_lib-0.7.5/src/mmisp/lib/logging.py

@@ -1,58 +0,0 @@
-from typing import Self
-
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from ..db.models.log import Log
-from ..db.models.workflow import Workflow
-
-
-class ApplicationLogger:
-    db_session: AsyncSession
-
-    def __init__(self: Self, db_session: AsyncSession) -> None:
-        self.db_session = db_session
-
-    def log_workflow_debug_message(self: Self, workflow: Workflow, message: str) -> Log | None:
-        """
-        If debugging is enabled, logs a message for the given workflow.
-
-        Arguments:
-            workflow:       The workflow a log entry is created for.
-            message:        The message to log.
-        """
-
-        if workflow.debug_enabled:
-            log_entry = self.__create_workflow(workflow.id, message)
-            self.db_session.add(log_entry)
-
-            return log_entry
-
-        return None
-
-    def log_workflow_execution_error(self: Self, workflow: Workflow, message: str) -> Log:
-        """
-        Logs a message in case of an error for the given workflow.
-
-        Arguments:
-            workflow:       The workflow a log entry is created for.
-            message:        The message to log.
-        """
-
-        log_entry = self.__create_workflow(workflow.id, message)
-        self.db_session.add(log_entry)
-
-        return log_entry
-
-    def __create_workflow(self: Self, id: int, message: str) -> Workflow:
-        return Log(
-            title=message,
-            action="execute_workflow",
-            model="Workflow",
-            model_id=id,
-            user_id=0,
-            email="SYSTEM",
-            org="SYSTEM",
-            description="",
-            change="",
-            ip="",
-        )