mmisp-lib 0.4.0__tar.gz → 0.6.0__tar.gz

{mmisp_lib-0.4.0 → mmisp_lib-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mmisp-lib
-Version: 0.4.0
+Version: 0.6.0
 Requires-Python: >=3.11.0
 Description-Content-Type: text/markdown
 License-File: LICENSE

{mmisp_lib-0.4.0 → mmisp_lib-0.6.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mmisp-lib"
-version = "0.4.0"
+version = "0.6.0"
 description = ""
 authors = []
 readme = "README.md"

{mmisp_lib-0.4.0 → mmisp_lib-0.6.0}/src/mmisp/api_schemas/attributes.py

@@ -165,29 +165,24 @@ class RestoreAttributeResponse(BaseModel):
 
 class GetDescribeTypesAttributes(BaseModel):
     sane_defaults: dict = {}
-    for k, v in to_ids.items():
+    for _k, _v in to_ids.items():
         sane_defaults.update(
             {
-                k: {
-                    "default_category": default_category[k],
-                    "to_ids": v,
+                _k: {
+                    "default_category": default_category[_k],
+                    "to_ids": "1" if _v else "0",
                 }
             }
         )
 
     types: list[str] = list(mapper_val_safe_clsname.keys())
-
     categories: list[str] = [member.value for member in AttributeCategories]
-
     category_type_mappings: dict = inverted_categories
 
 
 class GetDescribeTypesResponse(BaseModel):
     result: GetDescribeTypesAttributes
 
-    class Config:
-        orm_mode = True
-
 
 class GetAttributeAttributes(BaseModel):
     id: str
@@ -414,28 +409,12 @@ class AddAttributeBody(BaseModel):
 
 GetAttributeStatisticsTypesResponseAttrs = {x: Field(default=None) for x in mapper_val_safe_clsname.keys()}
 GetAttributeStatisticsTypesResponseAttrs["__annotations__"] = {x: str | None for x in mapper_val_safe_clsname.keys()}
-GetAttributeStatisticsTypesResponse = type(
+GetAttributeStatisticsTypesResponse = type(  # type: ignore
     "GetAttributeStatisticsTypesResponse", (BaseModel,), GetAttributeStatisticsTypesResponseAttrs
 )
 
-
-class GetAttributeStatisticsCategoriesResponse(BaseModel):
-    antivirus_detection: str = Field(alias="Antivirus detection")
-    artifacts_dropped: str = Field(alias="Artifacts dropped")
-    attribution: str = Field(alias="Attribution")
-    external_analysis: str = Field(alias="External analysis")
-    financial_fraud: str = Field(alias="Financial fraud")
-    internal_reference: str = Field(alias="Internal reference")
-    network_activity: str = Field(alias="Network activity")
-    other: str = Field(alias="Other")
-    payload_delivery: str = Field(alias="Payload delivery")
-    payload_installation: str = Field(alias="Payload installation")
-    payload_type: str = Field(alias="Payload type")
-    persistence_mechanism: str = Field(alias="Persistence mechanism")
-    person: str = Field(alias="Person")
-    social_network: str = Field(alias="Social network")
-    support__tool: str = Field(alias="Support Tool")
-    targeting_data: str = Field(alias="Targeting data")
-
-    class Config:
-        orm_mode = True
+GetAttributeStatisticsCategoriesResponseAttrs = {x.value: Field(default=None) for x in AttributeCategories}
+GetAttributeStatisticsCategoriesResponseAttrs["__annotations__"] = {x.value: str | None for x in AttributeCategories}
+GetAttributeStatisticsCategoriesResponse = type(  # type: ignore
+    "GetAttributeStatisticsCategoriesResponse", (BaseModel,), GetAttributeStatisticsCategoriesResponseAttrs
+)

{mmisp_lib-0.4.0 → mmisp_lib-0.6.0}/src/mmisp/api_schemas/galaxies.py

@@ -2,8 +2,8 @@ from typing import List
 
 from pydantic import BaseModel
 
-import mmisp.api_schemas.organisations
-from mmisp.api_schemas.events import AddEditGetEventGalaxyClusterRelation, GetAllEventsGalaxyClusterGalaxy
+from mmisp.api_schemas.galaxy_clusters import GetGalaxyClusterResponse
+from mmisp.api_schemas.galaxy_common import GetAllSearchGalaxiesAttributes
 
 
 class SearchGalaxiesBody(BaseModel):
@@ -38,31 +38,6 @@ class ExportGalaxyGalaxyElement(BaseModel):
     value: str
 
 
-class GetGalaxyClusterResponse(BaseModel):
-    id: str | None = None
-    uuid: str | None = None
-    collection_uuid: str
-    type: str
-    value: str
-    tag_name: str
-    description: str
-    galaxy_id: str
-    source: str
-    authors: list[str]
-    version: str
-    distribution: str
-    sharing_group_id: str
-    org_id: str
-    orgc_id: str
-    default: bool
-    locked: bool
-    extends_uuid: str
-    extends_version: str
-    published: bool
-    deleted: bool
-    GalaxyElement: list[ExportGalaxyGalaxyElement]
-
-
 class ImportGalaxyBody(BaseModel):
     GalaxyCluster: GetGalaxyClusterResponse
     Galaxy: ImportGalaxyGalaxy
@@ -71,20 +46,6 @@ class ImportGalaxyBody(BaseModel):
         orm_mode = True
 
 
-class GetAllSearchGalaxiesAttributes(BaseModel):
-    id: str
-    uuid: str
-    name: str
-    type: str
-    description: str
-    version: str
-    icon: str
-    namespace: str
-    kill_chain_order: str | None = None
-    enabled: bool
-    local_only: bool
-
-
 class GetAllSearchGalaxiesResponse(BaseModel):
     Galaxy: GetAllSearchGalaxiesAttributes
 
@@ -115,38 +76,6 @@ class GalaxySchema(BaseModel):
         orm_mode = True
 
 
-class ExportGalaxyClusterResponse(BaseModel):
-    id: str
-    uuid: str
-    collection_uuid: str
-    type: str
-    value: str
-    tag_name: str
-    description: str
-    galaxy_id: str
-    source: str
-    authors: list[str]
-    version: str
-    distribution: str
-    sharing_group_id: str
-    org_id: str
-    orgc_id: str
-    default: bool
-    locked: bool
-    extends_uuid: str
-    extends_version: str
-    published: bool
-    deleted: bool
-    GalaxyElement: list[ExportGalaxyGalaxyElement]
-    Galaxy: GetAllEventsGalaxyClusterGalaxy
-    GalaxyClusterRelation: list[AddEditGetEventGalaxyClusterRelation] = []
-    Org: mmisp.api_schemas.organisations.Organisation
-    Orgc: mmisp.api_schemas.organisations.Organisation
-
-    class Config:
-        orm_mode = True
-
-
 class ExportGalaxyAttributes(BaseModel):
     default: bool
     custom: bool | None = None

{mmisp_lib-0.4.0 → mmisp_lib-0.6.0}/src/mmisp/api_schemas/galaxy_clusters.py

@@ -1,10 +1,17 @@
+from datetime import datetime
+
 from pydantic import BaseModel
 
-from mmisp.api_schemas.events.add_edit_get_event_response import AddEditGetEventGalaxyClusterRelation
-from mmisp.api_schemas.events.get_all_events_response import GetAllEventsGalaxyClusterGalaxy
-from mmisp.api_schemas.galaxies import ExportGalaxyGalaxyElement, GetAllSearchGalaxiesAttributes
-from mmisp.api_schemas.organisations import GetOrganisationResponse
-from mmisp.api_schemas.organisations.organisation import Organisation
+from mmisp.api_schemas.events import AddEditGetEventGalaxyClusterRelation, GetAllEventsGalaxyClusterGalaxy
+from mmisp.api_schemas.galaxy_common import GetAllSearchGalaxiesAttributes
+from mmisp.api_schemas.organisations import GetOrganisationResponse, Organisation
+
+
+class ExportGalaxyGalaxyElement(BaseModel):
+    id: str | None = None
+    galaxy_cluster_id: str | None = None
+    key: str
+    value: str
 
 
 class GetGalaxyClusterResponse(BaseModel):
@@ -25,14 +32,23 @@ class GetGalaxyClusterResponse(BaseModel):
     orgc_id: str
     default: bool
     locked: bool
-    extends_uuid: str
+    extends_uuid: str | None = None
     extends_version: str
     published: bool
     deleted: bool
-    Galaxy: GetAllSearchGalaxiesAttributes
+    Galaxy: GetAllSearchGalaxiesAttributes | None = None
     GalaxyElement: list[ExportGalaxyGalaxyElement]
-    Org: GetOrganisationResponse
-    Orgc: GetOrganisationResponse
+    GalaxyClusterRelation: list = []
+    RelationshipInbound: list = []
+    Org: GetOrganisationResponse | None = None
+    Orgc: GetOrganisationResponse | None = None
+
+
+class GalaxyClusterResponse(BaseModel):
+    GalaxyCluster: GetGalaxyClusterResponse
+
+    class Config:
+        json_encoders = {datetime: lambda v: v.strftime("%Y-%m-%d %H:%M:%S")}
 
 
 class ExportGalaxyClusterResponse(BaseModel):

mmisp_lib-0.6.0/src/mmisp/api_schemas/galaxy_common.py

@@ -0,0 +1,15 @@
+from pydantic import BaseModel
+
+
+class GetAllSearchGalaxiesAttributes(BaseModel):
+    id: str
+    uuid: str
+    name: str
+    type: str
+    description: str
+    version: str
+    icon: str
+    namespace: str
+    kill_chain_order: str | None = None
+    enabled: bool
+    local_only: bool

{mmisp_lib-0.4.0 → mmisp_lib-0.6.0}/src/mmisp/api_schemas/organisations.py

@@ -3,17 +3,20 @@ from datetime import datetime
 from pydantic import BaseModel
 
 
-class Organisation(BaseModel):
-    id: str
-    name: str
+class BaseOrganisation(BaseModel):
+    id: str | None = None
+    name: str | None = None
+    nationality: str | None = None
+    sector: str | None = None
+    type: str | None = None
+    uuid: str | None = None
+
+
+class Organisation(BaseOrganisation):
     date_created: datetime
     date_modified: datetime
     description: str | None = None
-    type: str
-    nationality: str | None = None
-    sector: str | None = None
     created_by: str
-    uuid: str
     contacts: str | None = None
     local: bool
     """organisation gains access to the local instance, otherwise treated as external"""
@@ -27,14 +30,14 @@ class Organisation(BaseModel):
 class GetOrganisationResponse(BaseModel):
     id: str
     name: str
+    nationality: str | None = None
+    sector: str | None = None
+    type: str | None = None
+    uuid: str | None = None
     date_created: datetime
     date_modified: datetime
     description: str | None = None
-    type: str | None = None
-    nationality: str | None = None
-    sector: str | None = None
     created_by: str
-    uuid: str | None = None
     contacts: str | None = None
     local: bool
     restricted_to_domain: str | None = None

mmisp_lib-0.6.0/src/mmisp/api_schemas/servers.py

@@ -0,0 +1,138 @@
+from pydantic import BaseModel, Json
+
+from mmisp.api_schemas.organisations import BaseOrganisation
+
+
+class NOT_Filter(BaseModel):
+    NOT: list
+
+
+class OR_NOT_Filter(BaseModel):
+    OR: list
+    NOT: list
+
+
+class PullRulesFilter(BaseModel):
+    tags: OR_NOT_Filter
+    orgs: OR_NOT_Filter
+    type_attributes: NOT_Filter
+    type_objects: NOT_Filter
+    url_params: str
+
+
+class PushRulesFilter(BaseModel):
+    tags: OR_NOT_Filter
+    orgs: OR_NOT_Filter
+
+
+default_pull_rules = PullRulesFilter(
+    tags=OR_NOT_Filter(OR=[], NOT=[]),
+    orgs=OR_NOT_Filter(OR=[], NOT=[]),
+    type_attributes=NOT_Filter(NOT=[]),
+    type_objects=NOT_Filter(NOT=[]),
+    url_params="",
+)
+default_push_rules = PushRulesFilter(
+    tags=OR_NOT_Filter(OR=[], NOT=[]),
+    orgs=OR_NOT_Filter(OR=[], NOT=[]),
+)
+
+
+class AddServer(BaseModel):
+    url: str
+    name: str
+    remote_org_id: int
+    authkey: str
+    org_id: int | None = None
+    priority: int = 5
+    internal: bool = False
+    push: bool = False
+    pull: bool = False
+    pull_rules: Json[PullRulesFilter] | PullRulesFilter = default_pull_rules
+    push_rules: Json[PushRulesFilter] | PushRulesFilter = default_push_rules
+    push_galaxy_clusters: bool = False
+    caching_enabled: bool = False
+    unpublish_event: bool = False
+    publish_without_email: bool = False
+    self_signed: bool = False
+    skip_proxy: bool = False
+
+
+class EditServer(BaseModel):
+    name: str
+    url: str
+    priority: int
+    authkey: str
+    remote_org_id: int
+    internal: bool
+    push: bool
+    pull: bool
+    pull_rules: Json[PullRulesFilter] | PullRulesFilter = default_pull_rules
+    push_rules: Json[PushRulesFilter] | PushRulesFilter = default_push_rules
+    push_galaxy_clusters: bool
+    caching_enabled: bool
+    unpublish_event: bool
+    publish_without_email: bool
+    self_signed: bool
+    skip_proxy: bool
+
+
+class ServerResponseBase(BaseModel):
+    id: int
+    name: str
+    url: str
+    #    authkey: str
+    org_id: int | None = None
+    push: bool
+    pull: bool
+    cert_file: str | None
+    client_cert_file: str | None
+    lastpulledid: int | None
+    lastpushedid: int | None
+    organization: None
+    pull_analyst_data: bool
+    pull_rules: str
+    push_analyst_data: bool
+    push_rules: str
+    remove_missing_tags: bool
+    push_sightings: bool | None = None
+    push_galaxy_clusters: bool | None = None
+    pull_galaxy_clusters: bool | None = None
+    remote_org_id: int
+    publish_without_email: bool | None = None
+    unpublish_event: bool | None = None
+    self_signed: bool
+    internal: bool | None = None
+    skip_proxy: bool | None = None
+    caching_enabled: bool | None = None
+    priority: int | None = None
+
+
+class ServerResponse(ServerResponseBase):
+    cache_timestamp: bool = False
+
+
+class AddServerServer(ServerResponseBase):
+    authkey: str
+
+
+class AddServerResponse(BaseModel):
+    Server: AddServerServer
+
+
+class RemoveServer(BaseModel):
+    id: str
+    sharing_group_id: str
+    server_id: str
+    all_orgs: bool
+
+
+class GetRemoteServer(BaseModel):
+    Server: ServerResponse
+    Organisation: BaseOrganisation
+    RemoteOrg: BaseOrganisation
+    User: list
+
+
+class ServersGetVersion(BaseModel):
+    pass

{mmisp_lib-0.4.0 → mmisp_lib-0.6.0}/src/mmisp/db/models/server.py

@@ -1,4 +1,5 @@
 from sqlalchemy import Boolean, Integer, String, Text
+from sqlalchemy.orm import relationship
 
 from mmisp.db.mypy import Mapped, mapped_column
 
@@ -18,8 +19,10 @@ class Server(Base):
     push_sightings: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
     push_galaxy_clusters: Mapped[bool] = mapped_column(Boolean, default=False)
     pull_galaxy_clusters: Mapped[bool] = mapped_column(Boolean, default=False)
-    lastpulledid: Mapped[int] = mapped_column(Integer)
-    lastpushedid: Mapped[int] = mapped_column(Integer)
+    push_analyst_data: Mapped[bool] = mapped_column(Boolean, default=False)
+    pull_analyst_data: Mapped[bool] = mapped_column(Boolean, default=False)
+    last_pulled_id: Mapped[int] = mapped_column("lastpulledid", Integer)
+    last_pushed_id: Mapped[int] = mapped_column("lastpushedid", Integer)
     organization: Mapped[str] = mapped_column(String(10), default=None)
     remote_org_id: Mapped[int] = mapped_column(Integer, nullable=False, index=True)
     publish_without_email: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
@@ -33,3 +36,24 @@ class Server(Base):
     skip_proxy: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
     caching_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
     priority: Mapped[int] = mapped_column(Integer, nullable=False, default=0, index=True)
+    remove_missing_tags: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+
+    organisation = relationship(
+        "Organisation",
+        primaryjoin="Server.org_id == Organisation.id",
+        lazy="raise_on_sql",
+        foreign_keys="Server.org_id",
+    )  # type:ignore[assignment,var-annotated]
+    remote_organisation = relationship(
+        "Organisation",
+        primaryjoin="Server.remote_org_id == Organisation.id",
+        lazy="raise_on_sql",
+        foreign_keys="Server.remote_org_id",
+    )  # type:ignore[assignment,var-annotated]
+    users = relationship(
+        "User",
+        primaryjoin="Server.id == User.server_id",
+        foreign_keys="User.server_id",
+        back_populates="server",
+        lazy="raise_on_sql",
+    )

{mmisp_lib-0.4.0 → mmisp_lib-0.6.0}/src/mmisp/db/models/user.py

@@ -48,3 +48,10 @@ class User(Base):
 
     # Relationships
     org = relationship("Organisation", back_populates="users", lazy="raise_on_sql")
+    server = relationship(
+        "Server",
+        primaryjoin="Server.id == User.server_id",
+        foreign_keys="User.server_id",
+        back_populates="users",
+        lazy="raise_on_sql",
+    )

mmisp_lib-0.6.0/src/mmisp/db/print_changes.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+"""
+USAGE:
+    - Run `python -m mmisp.db.print_changes`
+"""
+
+import asyncio
+import pprint
+from collections import defaultdict
+
+from alembic.autogenerate import compare_metadata
+from alembic.migration import MigrationContext
+from sqlalchemy import MetaData
+
+import mmisp.db.all_models  # noqa
+
+from .database import Base, sessionmanager
+
+metadata = MetaData()
+
+
+def create_diff(conn) -> list:  # noqa
+    mc = MigrationContext.configure(conn)
+    diff = compare_metadata(mc, Base.metadata)  # type:ignore[attr-defined]
+    return diff
+
+
+async def print_changes() -> None:
+    changes = defaultdict(list)
+    sessionmanager.init()
+    assert sessionmanager._engine is not None
+    async with sessionmanager._engine.begin() as connection:
+        assert connection is not None
+        diff = await connection.run_sync(create_diff)
+        for elem in diff:
+            if isinstance(elem, list):
+                for inner_elem in elem:
+                    changes[inner_elem[0]].append(inner_elem)
+            else:
+                changes[elem[0]].append(elem)
+    pprint.pprint(changes, indent=2, width=20)
+    for k, v in changes.items():
+        print(k, len(v))
+
+
+asyncio.run(print_changes())