dasf-broker-django 0.0.1.dev0__py3-none-any.whl

dasf_broker/__init__.py

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: 2022-2025 Helmholtz-Zentrum hereon GmbH
+#
+# SPDX-License-Identifier: EUPL-1.2
+
+"""Django-based Message Broker for DASF
+
+A Django-based message broker for the Data Analytics Software Framework (DASF)
+"""
+
+from __future__ import annotations
+
+from . import _version
+
+__version__ = _version.get_versions()["version"]
+
+__author__ = "Philipp S. Sommer"
+__copyright__ = "Copyright (C) 2022 Helmholtz-Zentrum Hereon"
+__credits__ = [
+    "Philipp S. Sommer",
+]
+__license__ = "EUPL-1.2"
+
+__maintainer__ = "Helmholtz-Zentrum Hereon"
+__email__ = "hcdc_support@hereon.de"
+
+__status__ = "Pre-Alpha"

dasf_broker/_version.py

@@ -0,0 +1,21 @@
+
+# This file was generated by 'versioneer.py' (0.29) from
+# revision-control system data, or from the parent directory name of an
+# unpacked source archive. Distribution tarballs contain a pre-generated copy
+# of this file.
+
+import json
+
+version_json = '''
+{
+ "date": "2025-04-16T22:10:07+0200",
+ "dirty": false,
+ "error": null,
+ "full-revisionid": "b5fb8bd0d75756eb7c531bba7b50a9d73401626b",
+ "version": "0.0.1.dev0"
+}
+'''  # END VERSION_JSON
+
+
+def get_versions():
+    return json.loads(version_json)

dasf_broker/admin.py

@@ -0,0 +1,273 @@
+# SPDX-FileCopyrightText: 2022-2025 Helmholtz-Zentrum hereon GmbH
+#
+# SPDX-License-Identifier: EUPL-1.2
+
+"""Admin interfaces
+----------------
+
+This module defines the dasf-broker-django
+Admin interfaces.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, List, Optional, Tuple
+
+from django.contrib import admin  # noqa: F401
+from django.utils.timezone import now
+from guardian.admin import GuardedModelAdmin
+
+from dasf_broker import models  # noqa: F401
+
+if TYPE_CHECKING:
+    from django.db.models import QuerySet
+
+
+class ResponseTopicExistsFilter(admin.SimpleListFilter):
+    """Filter topics for response topics"""
+
+    title = "is response topic"
+
+    parameter_name = "responsetopic"
+
+    def lookups(self, request: Any, model_admin: Any) -> List[Tuple[Any, str]]:
+        return [("1", "Yes"), ("0", "No")]
+
+    def queryset(
+        self, request: Any, queryset: QuerySet[Any]
+    ) -> Optional[QuerySet[Any]]:
+        value = self.value()
+        if value == "1":
+            return queryset.filter(responsetopic__isnull=False)
+        elif value == "0":
+            return queryset.filter(responsetopic__isnull=True)
+        else:
+            return queryset
+
+
+class AvailabilityFilter(admin.SimpleListFilter):
+    """Filter topics based on their status."""
+
+    title = "Availability"
+
+    parameter_name = "last_pong"
+
+    def lookups(self, request: Any, model_admin: Any) -> List[Tuple[Any, str]]:
+        return [
+            ("offline", "Offline"),
+            ("online", "Online"),
+            ("unknown", "Unknown"),
+        ]
+
+    def queryset(  # type: ignore
+        self,
+        request: Any,
+        queryset: models.BrokerTopicQuerySet,  # type: ignore
+    ) -> Optional[QuerySet[Any]]:
+        value = self.value()
+        if value == "offline":
+            return queryset.filter_offline()
+        elif value == "online":
+            return queryset.filter_online()
+        elif value == "unknown":
+            return queryset.filter_unknown_availability()
+        else:
+            return queryset
+
+
+@admin.action(description="Ping topics")
+def ping_topics(modeladmin, request, queryset):
+    """Ping the broker topics."""
+    for topic in queryset.filter(supports_dasf=True):
+        topic.ping()
+
+
+@admin.action(description="Garbage collect topics")
+def garbage_collect(modeladmin, request, queryset):
+    queryset.filter(garbage_collect_on__lte=now()).delete()
+
+
+@admin.register(models.BrokerTopic)
+class BrokerTopicAdmin(GuardedModelAdmin):
+    """An admin for :model:`dasf_broker.BrokerTopic"""
+
+    search_fields = [
+        "slug__icontains",
+    ]
+
+    list_display = [
+        "slug",
+        "get_consumers",
+        "get_producers",
+        "is_public",
+        "is_response_topic",
+        "get_availability",
+        "pending_messages",
+        "stored_messages",
+        "stored_responses",
+    ]
+
+    list_filter = [
+        "is_public",
+        "date_created",
+        ResponseTopicExistsFilter,
+        AvailabilityFilter,
+    ]
+
+    actions = [ping_topics, garbage_collect]
+
+    @admin.display(description="Consumer(s)")  # type: ignore
+    def get_consumers(self, obj: models.BrokerTopic) -> str:
+        return ", ".join(map(str, obj.consumers))[:60]
+
+    @admin.display(description="Producer(s)")  # type: ignore
+    def get_producers(self, obj: models.BrokerTopic) -> str:
+        return ", ".join(map(str, obj.producers))[:60]
+
+    @admin.display(boolean=True)  # type: ignore
+    def is_response_topic(self, obj: models.BrokerTopic) -> bool:
+        return obj.is_response_topic
+
+    @admin.display(description="Online?", boolean=True)  # type: ignore
+    def get_availability(self, obj: models.BrokerTopic) -> Optional[bool]:
+        return obj.availability
+
+    def pending_messages(self, obj: models.BrokerTopic) -> str:
+        """Get the number of messages that have not yet been delivered"""
+        return str(obj.brokermessage_set.filter(delivered_to=None).count())
+
+    def stored_messages(self, obj: models.BrokerTopic) -> str:
+        """Get the number of messages in the database."""
+        return str(obj.brokermessage_set.count())
+
+    def stored_responses(self, obj: models.BrokerTopic) -> str:
+        """Get the number of messages in the database."""
+        if obj.is_response_topic:
+            return "-"
+        else:
+            return str(
+                models.BrokerMessage.objects.filter(
+                    topic__responsetopic__is_response_for=obj
+                ).count()
+            )
+
+
+@admin.register(models.ResponseTopic)
+class ResponseTopicAdmin(BrokerTopicAdmin):
+    """An admin for response topics."""
+
+    search_fields = BrokerTopicAdmin.search_fields + [
+        "is_response_for__slug__icontains"
+    ]
+
+    list_display = BrokerTopicAdmin.list_display[:-1]
+
+    list_filter = [
+        "is_public",
+        "date_created",
+    ]
+
+
+@admin.action(description="Send messages")
+def send_messages(modeladmin, request, queryset):
+    for message in queryset:
+        message.send()
+
+
+@admin.action(description="Ping topics")
+def ping_message_topics(modeladmin, request, queryset):
+    pks = queryset.values_list("topic__pk", flat=True)
+    for topic in models.BrokerTopic.objects.filter(
+        pk__in=pks, supports_dasf=True
+    ):
+        topic.ping()
+
+
+class IsResponseMessageFilter(admin.SimpleListFilter):
+    """Filter broker messages to a response topic"""
+
+    title = "is response message"
+
+    parameter_name = "topic"
+
+    def lookups(self, request: Any, model_admin: Any) -> List[Tuple[Any, str]]:
+        return [("1", "Yes"), ("0", "No")]
+
+    def queryset(
+        self, request: Any, queryset: QuerySet[Any]
+    ) -> Optional[QuerySet[Any]]:
+        value = self.value()
+        if value == "1":
+            return queryset.filter(topic__responsetopic__isnull=False)
+        elif value == "0":
+            return queryset.filter(topic__responsetopic__isnull=True)
+        else:
+            return queryset
+
+
+class HasBeenDeliveredFilter(admin.SimpleListFilter):
+    """Filter broker messages by their delivery state"""
+
+    title = "has been delivered"
+
+    parameter_name = "delivered_to"
+
+    def lookups(self, request: Any, model_admin: Any) -> List[Tuple[Any, str]]:
+        return [("1", "Yes"), ("0", "No")]
+
+    def queryset(
+        self, request: Any, queryset: QuerySet[Any]
+    ) -> Optional[QuerySet[Any]]:
+        value = self.value()
+        if value == "1":
+            return queryset.filter(delivered_to=True)
+        elif value == "0":
+            return queryset.filter(delivered_to=None)
+        else:
+            return queryset
+
+
+@admin.register(models.BrokerMessage)
+class BrokerMessageAdmin(GuardedModelAdmin):
+    """An admin for a broker message"""
+
+    search_fields = [
+        "topic__slug__icontains",
+        "topic__responsetopic__is_response_for__slug__icontains",
+        "user__username__icontains",
+        "user__email__icontains",
+        "user__first_name__icontains",
+        "user__last_name__icontains",
+    ]
+
+    list_display = [
+        "message_id",
+        "topic",
+        "topic_availability",
+        "is_response",
+        "user",
+        "delivered",
+        "date_created",
+    ]
+
+    list_filter = [
+        "date_created",
+        IsResponseMessageFilter,
+        HasBeenDeliveredFilter,
+    ]
+
+    readonly_fields = ["content"]
+
+    actions = [send_messages, ping_message_topics]
+
+    @admin.display(boolean=True)  # type: ignore
+    def delivered(self, obj: models.BrokerMessage) -> bool:
+        return obj.delivered_to.exists()
+
+    @admin.display(boolean=True)  # type: ignore
+    def is_response(self, obj: models.BrokerMessage) -> bool:
+        return obj.is_response
+
+    @admin.display(boolean=True, description="Topic online?")
+    def topic_availability(self, obj: models.BrokerMessage) -> Optional[bool]:
+        return obj.topic.availability

dasf_broker/app_settings.py

@@ -0,0 +1,133 @@
+# SPDX-FileCopyrightText: 2022-2025 Helmholtz-Zentrum hereon GmbH
+#
+# SPDX-License-Identifier: EUPL-1.2
+
+"""App settings
+------------
+
+This module defines the settings options for the
+``dasf-broker-django`` app.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Optional
+
+from django.conf import settings  # noqa: F401
+
+#: Create new topics on message
+#:
+#: This flag controls if new topics are created when a message comes from a
+#: producer. If False, messages with non-existing topics are ignored.
+#:
+#: Note that a user also needs the `dasf_broker.add_BrokerTopic`
+#: permission to create topics.
+DASF_CREATE_TOPIC_ON_MESSAGE: bool = getattr(
+    settings, "DASF_CREATE_TOPIC_ON_MESSAGE", True
+)
+
+
+class StoreMessageOptions(str, Enum):
+    DISABLED = "disabled"
+    CACHE = "cache"
+    CACHEALL = "cacheall"
+    STORE = "store"
+
+
+#: Shall the messages be stored?
+#:
+#: This flag controls whether the message broker caches messages from producers
+#: until they are consumed by the consumer. This is useful if the consumer
+#: looses connection to the server. This settings can take three different
+#: values:
+#:
+#: ``"disabled"``
+#:     The message is not stored at all
+#: ``"cache"``
+#:     The message is stored and removed ones one of the potential `consumers`
+#:     acknowledges the message
+#: ``"cacheall"``
+#:     The message is stored and removed ones **all** `consumers` acknowledged
+#:     the message
+#: ``"store"``
+#:     The message and response topics are stored forever and are not
+#:     automatically removed
+#:
+#: See Also
+#: --------
+#: DASF_STORE_SOURCE_MESSAGES
+#: DASF_STORE_RESPONSE_MESSAGES
+DASF_STORE_MESSAGES: StoreMessageOptions = StoreMessageOptions(
+    getattr(settings, "DASF_STORE_MESSAGES", StoreMessageOptions.CACHE)
+)
+
+#: Shall the source messages be stored?
+#:
+#: This flag controls whether the message broker caches messages from producers
+#: to topics that are **not** marked as response topic. If this setting is not
+#: set, we use the ``DASF_STORE_MESSAGES`` setting.
+DASF_STORE_SOURCE_MESSAGES: StoreMessageOptions = StoreMessageOptions(
+    getattr(settings, "DASF_STORE_SOURCE_MESSAGES", DASF_STORE_MESSAGES)
+)
+
+#: Shall the messages to response topics be stored?
+#:
+#: This flag controls whether the message broker caches messages from producers
+#: to topics that are marked as response topic. If this setting is not set, we
+#: use the ``DASF_STORE_MESSAGES`` setting.
+DASF_STORE_RESPONSE_MESSAGES: StoreMessageOptions = StoreMessageOptions(
+    getattr(settings, "DASF_STORE_RESPONSE_MESSAGES", DASF_STORE_MESSAGES)
+)
+
+#: URL route for the websocket
+#:
+#: This setting controls, where we expect to find the websockets.
+#: As there is no analog to :func:`django.urls.reverse` for channels, you
+#: should use this setting in your ``asgi.py`` file to include the routes
+#: of this package.
+#:
+#: Examples
+#: --------
+#: In your ``asgi.py`` file, include it like:
+#:
+#: .. code-block:: python
+#:
+#:     from channels.routing import ProtocolTypeRouter, URLRouter
+#:     from django.core.asgi import get_asgi_application
+#:     from channels.auth import AuthMiddlewareStack
+#:     import dasf_broker.routing as dasf_routing
+#:     from dasf_broker.app_settings import DASF_WEBSOCKET_URL_ROUTE
+#:
+#:     application = ProtocolTypeRouter(
+#:         {
+#:             "http": get_asgi_application(),
+#:             "websocket": AuthMiddlewareStack(
+#:                 URLRouter(
+#:                     [
+#:                         path(
+#:                             DASF_WEBSOCKET_URL_ROUTE,
+#:                             URLRouter(dasf_routing.websocket_urlpatterns),
+#:                         )
+#:                     ]
+#:                 )
+#:             ),
+#:         }
+#:     )
+DASF_WEBSOCKET_URL_ROUTE: str = getattr(
+    settings, "DASF_WEBSOCKET_URL_ROUTE", "ws/"
+)
+
+
+#: root URL to your application
+#:
+#: You can use this setting if you are behind a reverse proxy and the
+#: host names, etc. are not handled correctly.
+#:
+#: If you leave this empty, we will use the ``build_absolute_uri`` method
+#: of the http request.
+#:
+#: Examples
+#: --------
+#: A standard value for this would be ``http://localhost:8000``
+ROOT_URL: Optional[str] = None

dasf_broker/apps.py

@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: 2022-2025 Helmholtz-Zentrum hereon GmbH
+#
+# SPDX-License-Identifier: EUPL-1.2
+
+"""App config
+----------
+
+App config for the dasf_broker app.
+"""
+
+from django.apps import AppConfig
+
+
+class DjangoHelmholtzAaiConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "dasf_broker"

dasf_broker/constants.py

@@ -0,0 +1,31 @@
+# SPDX-FileCopyrightText: 2022-2025 Helmholtz-Zentrum hereon GmbH
+#
+# SPDX-License-Identifier: EUPL-1.2
+
+"""Constants within the DASF Framework
+-----------------------------------
+
+Constants from :mod:`demessaging.PulsarMessageConstants`.
+"""
+
+from enum import Enum
+
+
+class PropertyKeys(str, Enum):
+    REQUEST_CONTEXT = "requestContext"
+    RESPONSE_TOPIC = "response_topic"
+    REQUEST_MESSAGEID = "requestMessageId"
+    MESSAGE_TYPE = "messageType"
+    FRAGMENT = "fragment"
+    NUM_FRAGMENTS = "num_fragments"
+    STATUS = "status"
+
+
+class MessageType(str, Enum):
+    PING = "ping"
+    PONG = "pong"
+    REQUEST = "request"
+    RESPONSE = "response"
+    LOG = "log"
+    INFO = "info"
+    PROGRESS = "progress"