mmisp-lib 0.8.2__tar.gz → 0.8.3__tar.gz

{mmisp_lib-0.8.2 → mmisp_lib-0.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mmisp-lib
-Version: 0.8.2
+Version: 0.8.3
 Requires-Python: >=3.11.0
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -21,6 +21,7 @@ Requires-Dist: nanoid==2.0.0
 Requires-Dist: cryptography==42.0.5
 Requires-Dist: jinja2~=3.1.4
 Requires-Dist: fire
+Requires-Dist: packaging
 Provides-Extra: dev
 Requires-Dist: ruff>=0.3.7; extra == "dev"
 Requires-Dist: mypy==1.8.0; extra == "dev"

{mmisp_lib-0.8.2 → mmisp_lib-0.8.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mmisp-lib"
-version = "0.8.2"
+version = "0.8.3"
 description = ""
 authors = []
 readme = "README.md"
@@ -25,6 +25,7 @@ dependencies = [
   "cryptography==42.0.5",
   "jinja2~=3.1.4",
   "fire",
+  "packaging",
 ]
 
 

mmisp_lib-0.8.3/src/mmisp/api_schemas/object_templates.py

@@ -0,0 +1,50 @@
+from pydantic import BaseModel, Field
+from pydantic.types import UUID
+
+
+class ObjectTemplatesRequirements(BaseModel):
+    requiredOneOf: list[str] | None
+    required: list[str] | None
+
+
+class ObjectTemplate(BaseModel):
+    class Config:
+        allow_population_by_field_name = True
+
+    id: int
+    user_id: int
+    org_id: int
+    uuid: UUID
+    name: str
+    meta_category: str = Field(..., alias="meta-category")
+    description: str
+    version: str
+    requirements: ObjectTemplatesRequirements
+    fixed: bool
+    active: bool
+
+
+class ObjectTemplateElement(BaseModel):
+    class Config:
+        allow_population_by_field_name = True
+
+    id: int
+    object_template_id: int
+    object_relation: str
+    type: str  # AttributeType?
+    ui_priority: int = Field(..., alias="ui-priority")
+    categories: list
+    sane_default: list
+    values_list: list
+    description: str
+    disable_correlation: bool | None
+    multiple: bool
+
+
+class RespObjectTemplateView(BaseModel):
+    ObjectTemplate: ObjectTemplate
+    ObjectTemplateElement: list[ObjectTemplateElement]
+
+
+class RespItemObjectTemplateIndex(BaseModel):
+    ObjectTemplate: ObjectTemplate

{mmisp_lib-0.8.2 → mmisp_lib-0.8.3}/src/mmisp/db/database.py

@@ -1,6 +1,7 @@
 import contextlib
 import time
 from collections.abc import AsyncIterator
+from contextvars import ContextVar
 from typing import Self, TypeAlias
 
 from sqlalchemy.engine.url import make_url
@@ -20,6 +21,8 @@ Base = declarative_base()
 
 _no_database: str = "DatabaseSessionManager is not initialized"
 
+dry_run: ContextVar[bool] = ContextVar("dry_run", default=False)
+
 
 class DatabaseSessionManager:
     def __init__(self: Self, db_url: str = config.DATABASE_URL) -> None:
@@ -108,6 +111,8 @@ class DatabaseSessionManager:
 async def get_db() -> AsyncIterator[Session]:
     async with sessionmanager.session() as session:
         yield session
+        if dry_run.get():
+            await session.rollback()
 
 
 async def create_all_models() -> None:

{mmisp_lib-0.8.2 → mmisp_lib-0.8.3}/src/mmisp/db/mixins.py

@@ -18,6 +18,8 @@ class DictMixin:
         if omit is None:
             omit = set()
 
+        print(self.__mapper__.c.keys())
+
         unloaded = inspect(self).unloaded
         d = {}
         for key in self.__mapper__.c.keys():  # type:ignore[attr-defined]

{mmisp_lib-0.8.2 → mmisp_lib-0.8.3}/src/mmisp/db/models/attribute.py

@@ -40,7 +40,7 @@ class Attribute(Base, DictMixin):
         Integer, ForeignKey("events.id", ondelete="CASCADE"), nullable=False, index=True
     )
     object_id: Mapped[int] = mapped_column(Integer, nullable=False, default=0, index=True)
-    object_relation: Mapped[str] = mapped_column(String(255), index=True)
+    object_relation: Mapped[str | None] = mapped_column(String(255), index=True)
     category: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
     type: Mapped[str] = mapped_column(String(100), nullable=False, index=True)
     value1: Mapped[str] = mapped_column(Text, nullable=False)
@@ -49,11 +49,11 @@ class Attribute(Base, DictMixin):
     timestamp: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
     distribution: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
     sharing_group_id: Mapped[int] = mapped_column(Integer, index=True, default=0)
-    comment: Mapped[str] = mapped_column(Text)
+    comment: Mapped[str | None] = mapped_column(Text)
     deleted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
     disable_correlation: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
-    first_seen: Mapped[int] = mapped_column(BigInteger, index=True)
-    last_seen: Mapped[int] = mapped_column(BigInteger, index=True)
+    first_seen: Mapped[int | None] = mapped_column(BigInteger, index=True)
+    last_seen: Mapped[int | None] = mapped_column(BigInteger, index=True)
 
     event = relationship("Event", back_populates="attributes", lazy="joined")  # type:ignore[var-annotated]
     mispobject = relationship(

{mmisp_lib-0.8.2 → mmisp_lib-0.8.3}/src/mmisp/db/models/auth_key.py

@@ -19,7 +19,7 @@ class AuthKey(Base):
     created: Mapped[int] = mapped_column(Integer, nullable=False, default=time)
     expiration: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
     read_only: Mapped[bool] = mapped_column(Boolean, nullable=False, default=0)
-    comment: Mapped[str] = mapped_column(String(255))
-    allowed_ips: Mapped[str] = mapped_column(String(255))
-    unique_ips: Mapped[str] = mapped_column(String(255))
+    comment: Mapped[str | None] = mapped_column(String(255))
+    allowed_ips: Mapped[str | None] = mapped_column(String(255))
+    unique_ips: Mapped[str | None] = mapped_column(String(255))
     user_id: Mapped[int] = mapped_column(Integer, ForeignKey("users.id"), nullable=False)

{mmisp_lib-0.8.2 → mmisp_lib-0.8.3}/src/mmisp/db/models/event.py

@@ -37,7 +37,7 @@ class Event(Base):
     sighting_timestamp: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
     disable_correlation: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
     extends_uuid: Mapped[str] = mapped_column(String(40), default="", index=True)
-    protected: Mapped[bool] = mapped_column(Boolean)
+    protected: Mapped[bool] = mapped_column(Boolean, default=False)
 
     attributes = relationship("Attribute", back_populates="event")  # type:ignore[assignment,var-annotated]
     mispobjects = relationship("Object", back_populates="event")  # type:ignore[assignment,var-annotated]

{mmisp_lib-0.8.2 → mmisp_lib-0.8.3}/src/mmisp/db/models/feed.py

@@ -11,7 +11,7 @@ class Feed(Base):
     name: Mapped[str] = mapped_column(String(255), nullable=False)
     provider: Mapped[str] = mapped_column(String(255), nullable=False)
     url: Mapped[str] = mapped_column(String(255), nullable=False)
-    rules: Mapped[str] = mapped_column(Text, default=None)
+    rules: Mapped[str | None] = mapped_column(Text, default=None)
     enabled: Mapped[bool] = mapped_column(Boolean, default=False)
     distribution: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
     sharing_group_id: Mapped[int] = mapped_column(Integer, nullable=False, default=0, index=True)
@@ -23,11 +23,11 @@ class Feed(Base):
     event_id: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
     publish: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
     override_ids: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
-    settings: Mapped[str] = mapped_column(Text)
+    settings: Mapped[str | None] = mapped_column(Text)
     input_source: Mapped[str] = mapped_column(String(255), nullable=False, default="network", index=True)
     delete_local_file: Mapped[bool] = mapped_column(Boolean, default=False)
     lookup_visible: Mapped[bool] = mapped_column(Boolean, default=False)
-    headers: Mapped[str] = mapped_column(Text)
+    headers: Mapped[str | None] = mapped_column(Text)
     caching_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
     force_to_ids: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
     orgc_id: Mapped[int] = mapped_column(Integer, nullable=False, default=0, index=True)

{mmisp_lib-0.8.2 → mmisp_lib-0.8.3}/src/mmisp/db/models/galaxy.py

@@ -21,7 +21,7 @@ class Galaxy(Base, DictMixin, UpdateMixin):
     version: Mapped[str] = mapped_column(String(255), nullable=False)
     icon: Mapped[str] = mapped_column(String(255), nullable=False, default="")
     namespace: Mapped[str] = mapped_column(String(255), nullable=False, default="misp", index=True)
-    kill_chain_order: Mapped[str] = mapped_column(String(255))
+    kill_chain_order: Mapped[str | None] = mapped_column(String(255))
     """must be serialized"""
     enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
     local_only: Mapped[bool] = mapped_column(Boolean, default=False)

{mmisp_lib-0.8.2 → mmisp_lib-0.8.3}/src/mmisp/db/models/galaxy_cluster.py

@@ -33,8 +33,8 @@ class GalaxyCluster(Base, UpdateMixin, DictMixin):
     orgc_id: Mapped[int] = mapped_column(Integer, nullable=False, index=True, default=0)
     default: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
     locked: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
-    extends_uuid: Mapped[str] = mapped_column(String(40), index=True)
-    extends_version: Mapped[int] = mapped_column(Integer, index=True, nullable=True, default=None)
+    extends_uuid: Mapped[str | None] = mapped_column(String(40), index=True)
+    extends_version: Mapped[int | None] = mapped_column(Integer, index=True, nullable=True, default=None)
     published: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
     deleted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 

{mmisp_lib-0.8.2 → mmisp_lib-0.8.3}/src/mmisp/db/models/identity_provider.py

@@ -17,7 +17,7 @@ class OIDCIdentityProvider(Base):
     base_url: Mapped[str] = mapped_column(String(255), nullable=False)
     client_id: Mapped[str] = mapped_column(String(255), nullable=False)
     client_secret: Mapped[str] = mapped_column(String(255), nullable=False)
-    scope: Mapped[str] = mapped_column(String(255))
+    scope: Mapped[str] = mapped_column(String(255), default="openid")
     """Possibility to add more scopes to be requested from the idp in addition to the default scopes,
     currently not used."""
     created: Mapped[DateTime] = mapped_column(DateTime, default=datetime.utcnow)

{mmisp_lib-0.8.2 → mmisp_lib-0.8.3}/src/mmisp/db/models/object.py

@@ -1,9 +1,13 @@
-from sqlalchemy import Boolean, ForeignKey, Integer, String
+from typing import Optional
+
+from sqlalchemy import Boolean, ForeignKey, Integer, String, Text
 from sqlalchemy.orm import relationship
 
 from mmisp.db.database import Base
+from mmisp.db.list_json_type import DBListJson
 from mmisp.db.mixins import DictMixin
 from mmisp.db.mypy import Mapped, mapped_column
+from mmisp.db.object_json_type import DBObjectJson
 from mmisp.lib.uuid import uuid
 
 
@@ -13,7 +17,7 @@ class Object(Base, DictMixin):
     id: Mapped[int] = mapped_column(Integer, primary_key=True, nullable=False)
     uuid: Mapped[str] = mapped_column(String(255), unique=True, default=uuid, index=True)
     name: Mapped[str] = mapped_column(String(255), index=True)
-    meta_category: Mapped[str] = mapped_column("meta-category", String(255), index=True, key="meta_category")
+    meta_category: Mapped[str] = mapped_column("meta-category", String(255), index=True)
     description: Mapped[str] = mapped_column(String(255))
     template_uuid: Mapped[str] = mapped_column(String(255), index=True, default=None)
     template_version: Mapped[int] = mapped_column(Integer, index=True, nullable=False)
@@ -44,12 +48,41 @@ class ObjectTemplate(Base, DictMixin):
     __tablename__ = "object_templates"
 
     id: Mapped[int] = mapped_column(Integer, primary_key=True, nullable=False)
-    uuid: Mapped[str] = mapped_column(String(255), unique=True, default=uuid, index=True)
-    name: Mapped[str] = mapped_column(String(255), index=True)
     user_id: Mapped[int] = mapped_column(Integer, ForeignKey("users.id"), index=True, nullable=False)
     org_id: Mapped[int] = mapped_column(Integer, ForeignKey("organisations.id"), index=True, nullable=False)
-    description: Mapped[str] = mapped_column(String(255))
+    uuid: Mapped[str] = mapped_column(String(255), unique=True, default=uuid, index=True)
+    name: Mapped[Optional[str]] = mapped_column(String(255), nullable=True, index=True)
+    meta_category: Mapped[Optional[str]] = mapped_column(
+        "meta-category", String(255), nullable=True, index=True, key="meta_category"
+    )
+    description: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
     version: Mapped[int] = mapped_column(Integer, nullable=False)
-    requirements: Mapped[str] = mapped_column(String(255))
+    requirements: Mapped[dict | None] = mapped_column(DBObjectJson, nullable=True)
     fixed: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
     active: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+
+    elements: Mapped[list["ObjectTemplateElement"]] = relationship(
+        "ObjectTemplateElement", back_populates="object_template", lazy="raise_on_sql"
+    )
+
+
+class ObjectTemplateElement(Base, DictMixin):
+    __tablename__ = "object_template_elements"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, nullable=False)
+    object_template_id: Mapped[int] = mapped_column(
+        Integer, ForeignKey(ObjectTemplate.id, ondelete="CASCADE"), nullable=False, index=True
+    )
+    object_relation: Mapped[Optional[str]] = mapped_column(String(255), nullable=True, index=True)
+    type: Mapped[Optional[str]] = mapped_column(Text, nullable=True, index=True)
+    ui_priority: Mapped[int] = mapped_column("ui-priority", Integer, nullable=False, key="ui_priority")
+    categories: Mapped[list[str] | None] = mapped_column(DBListJson, nullable=True)
+    sane_default: Mapped[list[str] | None] = mapped_column(DBListJson, nullable=True)
+    values_list: Mapped[list[str] | None] = mapped_column(DBListJson, nullable=True)
+    description: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+    disable_correlation: Mapped[Optional[bool]] = mapped_column(Boolean, nullable=True)
+    multiple: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+
+    object_template: Mapped[ObjectTemplate] = relationship(
+        ObjectTemplate, back_populates="elements", lazy="raise_on_sql"
+    )

{mmisp_lib-0.8.2 → mmisp_lib-0.8.3}/src/mmisp/db/models/server.py

@@ -22,17 +22,17 @@ class Server(Base, DictMixin):
     pull_galaxy_clusters: Mapped[bool] = mapped_column(Boolean, default=False)
     push_analyst_data: Mapped[bool] = mapped_column(Boolean, default=False)
     pull_analyst_data: Mapped[bool] = mapped_column(Boolean, default=False)
-    last_pulled_id: Mapped[int] = mapped_column("lastpulledid", Integer)
-    last_pushed_id: Mapped[int] = mapped_column("lastpushedid", Integer)
-    organization: Mapped[str] = mapped_column(String(10), default=None)
+    last_pulled_id: Mapped[int] = mapped_column("lastpulledid", Integer, nullable=True)
+    last_pushed_id: Mapped[int] = mapped_column("lastpushedid", Integer, nullable=True)
+    organization: Mapped[str | None] = mapped_column(String(10), default=None)
     remote_org_id: Mapped[int] = mapped_column(Integer, nullable=False, index=True)
     publish_without_email: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
     unpublish_event: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
     self_signed: Mapped[bool] = mapped_column(Boolean, nullable=False)
     pull_rules: Mapped[str] = mapped_column(Text, nullable=False)
     push_rules: Mapped[str] = mapped_column(Text, nullable=False)
-    cert_file: Mapped[str] = mapped_column(String(255))
-    client_cert_file: Mapped[str] = mapped_column(String(255))
+    cert_file: Mapped[str | None] = mapped_column(String(255))
+    client_cert_file: Mapped[str | None] = mapped_column(String(255))
     internal: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
     skip_proxy: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
     caching_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)

{mmisp_lib-0.8.2 → mmisp_lib-0.8.3}/src/mmisp/db/models/tag.py

@@ -15,7 +15,7 @@ class Tag(Base):
     org_id: Mapped[int] = mapped_column(Integer, nullable=False, default=0, index=True)
     user_id: Mapped[int] = mapped_column(Integer, nullable=False, default=0, index=True)
     hide_tag: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
-    numerical_value: Mapped[int] = mapped_column(Integer, index=True)
+    numerical_value: Mapped[int | None] = mapped_column(Integer, index=True)
     is_galaxy: Mapped[bool] = mapped_column(Boolean, default=False)
     is_custom_galaxy: Mapped[bool] = mapped_column(Boolean, default=False)
     local_only: Mapped[bool] = mapped_column(Boolean, default=False)

{mmisp_lib-0.8.2 → mmisp_lib-0.8.3}/src/mmisp/db/models/user.py

@@ -20,8 +20,8 @@ class User(Base):
     autoalert: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
     authkey: Mapped[str] = mapped_column(String(40), nullable=True, default=None)
     invited_by: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
-    gpgkey: Mapped[str] = mapped_column(Text)
-    certif_public: Mapped[str] = mapped_column(Text)
+    gpgkey: Mapped[str | None] = mapped_column(Text)
+    certif_public: Mapped[str | None] = mapped_column(Text)
     nids_sid: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
     termsaccepted: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
     newsread: Mapped[int] = mapped_column(Integer, default=0)
@@ -35,16 +35,16 @@ class User(Base):
     force_logout: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
     date_created: Mapped[int] = mapped_column(Integer, default=time)
     date_modified: Mapped[int] = mapped_column(Integer, default=time, onupdate=time)
-    sub: Mapped[str] = mapped_column(String(255), unique=True)
+    sub: Mapped[str] = mapped_column(String(255), unique=True, nullable=True)
     external_auth_required: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
-    external_auth_key: Mapped[str] = mapped_column(Text)
+    external_auth_key: Mapped[str] = mapped_column(Text, nullable=True)
     last_api_access: Mapped[int] = mapped_column(Integer, default=0)
     notification_daily: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
     notification_weekly: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
     notification_monthly: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
-    totp: Mapped[str] = mapped_column(String(255))
-    hotp_counter: Mapped[int] = mapped_column(Integer)
-    last_pw_change: Mapped[int] = mapped_column(BigInteger)
+    totp: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    hotp_counter: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    last_pw_change: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
 
     # Relationships
     org = relationship("Organisation", back_populates="users", lazy="raise_on_sql")

{mmisp_lib-0.8.2 → mmisp_lib-0.8.3}/src/mmisp/db/mypy.py

@@ -1,6 +1,11 @@
+import importlib.metadata
 from typing import TYPE_CHECKING
 
-if TYPE_CHECKING:
+from packaging.version import Version
+
+sa_version = Version(importlib.metadata.version("sqlalchemy"))
+
+if TYPE_CHECKING or sa_version >= Version("2.0"):
     from sqlalchemy.orm import Mapped, mapped_column  # type:ignore[attr-defined]
 else:
     from sqlalchemy import Column

mmisp_lib-0.8.3/src/mmisp/db/object_json_type.py

@@ -0,0 +1,34 @@
+import json
+from typing import Any, Self
+
+from sqlalchemy.engine import Dialect
+from sqlalchemy.types import Text, TypeDecorator
+
+
+class DBObjectJson(TypeDecorator):
+    impl = Text
+
+    def load_dialect_impl(self: Self, dialect: Dialect) -> Any:
+        return dialect.type_descriptor(Text)
+
+    def process_bind_param(self: Self, value: Any, dialect: Dialect) -> str | None:
+        """Handle value before getting into the DB"""
+        if value is None:
+            return None
+        if not isinstance(value, dict):
+            raise ValueError(f"this type should only be used for dicts, got {type(value)}")
+        return json.dumps(value)
+
+    def process_result_value(self: Self, value: Any, dialect: Dialect) -> dict | None:
+        """Handle values from the database"""
+        if value is None:
+            return None
+
+        res = json.loads(value)
+        print("Result is", res)
+        if res == []:
+            res = {}
+        if not isinstance(res, dict):
+            raise ValueError(f"this type should only be used for dicts got: {type(res)}")
+
+        return res