explicit-python-dlq 1.0.0__tar.gz

explicit_python_dlq-1.0.0/MANIFEST.in

@@ -0,0 +1,18 @@
+graft src
+
+global-exclude *.pyc
+global-exclude __pycache__
+
+prune src/testapp
+
+exclude pylintrc
+exclude mypy.ini
+exclude db.sqlite3
+exclude isort.cfg
+exclude .coveragerc
+exclude .coverage
+
+include README.md
+include version.conf
+include requirements/base.txt
+include requirements/prod.txt

explicit_python_dlq-1.0.0/PKG-INFO

@@ -0,0 +1,105 @@
+Metadata-Version: 2.4
+Name: explicit-python-dlq
+Version: 1.0.0
+Summary: Набор компонентов реализующих DQL для систем на базе explicit
+Author-email: "АО \"БАРС Груп\"" <education_dev@bars-open.ru>
+Classifier: Intended Audience :: Developers
+Classifier: Environment :: Web Environment
+Classifier: Natural Language :: Russian
+Classifier: Operating System :: OS Independent
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: explicit-python<3,>=2.2.1
+Provides-Extra: dev
+Requires-Dist: freezegun; extra == "dev"
+Provides-Extra: django
+Requires-Dist: explicit-python-django<2.0,>=1.0.2; extra == "django"
+Provides-Extra: rest
+Requires-Dist: djangorestframework>=3.13.0; extra == "rest"
+Requires-Dist: django_filter>=23.3; extra == "rest"
+
+# Набор компонентов реализующих очередь необработанных сообщений (Dead Letter Queue).
+
+## Пример подключения
+testapp/dlq/apps.py:
+```python
+from django.apps import AppConfig as AppConfigBase
+
+
+class AppConfig(AppConfigBase):
+    name = __package__
+
+    def ready(self):
+        self._configure_dlq()
+
+    def _configure_dlq(self):
+        from testapp.core import bus
+        from explicit.dlq.config import DLQConfig, configure_dlq
+        # реализация репозитория специфичная для используемого слоя хранения данных
+        from testapp.dlq.adapters.db import Repository
+        
+        # реализация обработчика сообщений, специфичного для приложения
+        from testapp.dlq.services.dispatch import dispatch_message
+
+        config = DLQConfig(
+            bus=bus,
+            repository=Repository(),
+            message_dispatcher=dispatch_message
+        )
+        # регистрация репозитория и регистрация обработчиков команд
+        configure_dlq(config)
+```
+
+testapp/dlq/services/dispatch.py:
+```python
+import json
+from explicit.contrib.messagebus.event_registry import Message
+
+from testapp.core import bus
+from testapp.core import event_registry
+
+def dispatch_message(raw_message_value: bytes, topic: str):
+    """Преобразует сообщение в событие и отправляет его в шину."""
+    message = json.loads(raw_message_value)
+
+    if event := event_registry.resolve(
+        Message(
+            topic=topic,
+            type=message.get('type'),
+            body=message
+        )
+    ):
+        bus.handle(event)
+
+````
+
+testapp/entrypoints/kafka.py:
+
+```python
+def bootstrap() -> None:
+    from explicit.dlq.infrastructure.handlers import RegisterInDLQOnFailure
+
+    from testapp.core import bus
+
+    from testapp.dlq.services.dispatch import dispatch_message
+    topics = ('test.foo.topic',)
+
+    for raw_message in adapter.subscribe(*topics):
+        with RegisterInDLQOnFailure(
+            bus=bus,
+            topic=raw_message.topic(),
+            raw_message_value=raw_message.value(),
+            raw_message_key=raw_message.key
+        ):
+            dispatch_message(raw_message.value(), raw_message.topic())
+```
+## Готовые компоненты (contrib)
+
+В пакете реализованы готовые к использованию компоненты:
+* Реализация абстрактного [хранилища сообщений](./src/explicit/dlq/contrib/django/README.md) на базе Django ORM
+* [REST API](./src/explicit/dlq/contrib/drf/README.md) для работы с хранилищем сообщений. Предназначен для совместной работы с реализацией хранилища сообщений django ORM.

explicit_python_dlq-1.0.0/README.md

@@ -0,0 +1,80 @@
+# Набор компонентов реализующих очередь необработанных сообщений (Dead Letter Queue).
+
+## Пример подключения
+testapp/dlq/apps.py:
+```python
+from django.apps import AppConfig as AppConfigBase
+
+
+class AppConfig(AppConfigBase):
+    name = __package__
+
+    def ready(self):
+        self._configure_dlq()
+
+    def _configure_dlq(self):
+        from testapp.core import bus
+        from explicit.dlq.config import DLQConfig, configure_dlq
+        # реализация репозитория специфичная для используемого слоя хранения данных
+        from testapp.dlq.adapters.db import Repository
+        
+        # реализация обработчика сообщений, специфичного для приложения
+        from testapp.dlq.services.dispatch import dispatch_message
+
+        config = DLQConfig(
+            bus=bus,
+            repository=Repository(),
+            message_dispatcher=dispatch_message
+        )
+        # регистрация репозитория и регистрация обработчиков команд
+        configure_dlq(config)
+```
+
+testapp/dlq/services/dispatch.py:
+```python
+import json
+from explicit.contrib.messagebus.event_registry import Message
+
+from testapp.core import bus
+from testapp.core import event_registry
+
+def dispatch_message(raw_message_value: bytes, topic: str):
+    """Преобразует сообщение в событие и отправляет его в шину."""
+    message = json.loads(raw_message_value)
+
+    if event := event_registry.resolve(
+        Message(
+            topic=topic,
+            type=message.get('type'),
+            body=message
+        )
+    ):
+        bus.handle(event)
+
+````
+
+testapp/entrypoints/kafka.py:
+
+```python
+def bootstrap() -> None:
+    from explicit.dlq.infrastructure.handlers import RegisterInDLQOnFailure
+
+    from testapp.core import bus
+
+    from testapp.dlq.services.dispatch import dispatch_message
+    topics = ('test.foo.topic',)
+
+    for raw_message in adapter.subscribe(*topics):
+        with RegisterInDLQOnFailure(
+            bus=bus,
+            topic=raw_message.topic(),
+            raw_message_value=raw_message.value(),
+            raw_message_key=raw_message.key
+        ):
+            dispatch_message(raw_message.value(), raw_message.topic())
+```
+## Готовые компоненты (contrib)
+
+В пакете реализованы готовые к использованию компоненты:
+* Реализация абстрактного [хранилища сообщений](./src/explicit/dlq/contrib/django/README.md) на базе Django ORM
+* [REST API](./src/explicit/dlq/contrib/drf/README.md) для работы с хранилищем сообщений. Предназначен для совместной работы с реализацией хранилища сообщений django ORM.

explicit_python_dlq-1.0.0/pyproject.toml

@@ -0,0 +1,132 @@
+[build-system]
+requires = [
+    "packaging>=24,<25",
+    "wheel>=0.45,<0.46",
+    "setuptools>=77,<78",
+    "setuptools-git-versioning>=2,<3",
+]
+
+build-backend = "setuptools.build_meta"
+
+[project]
+dynamic = ["version"]
+
+name = "explicit-python-dlq"
+
+authors = [
+  {name = "АО \"БАРС Груп\"", email = "education_dev@bars-open.ru"}
+]
+
+readme = "README.md"
+
+description = "Набор компонентов реализующих DQL для систем на базе explicit"
+
+classifiers=[
+    "Intended Audience :: Developers",
+    "Environment :: Web Environment",
+    "Natural Language :: Russian",
+    "Operating System :: OS Independent",
+    "Development Status :: 5 - Production/Stable",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+
+requires-python = ">=3.9"
+
+dependencies = [
+  "explicit-python>=2.2.1,<3",
+]
+
+[project.optional-dependencies]
+dev = [
+    "freezegun"
+]
+django = [
+    "explicit-python-django>=1.0.2,<2.0"
+]
+rest = [
+    "djangorestframework>=3.13.0",
+    "django_filter>=23.3",
+]
+
+
+[tool.setuptools-git-versioning]
+enabled = true
+
+[tool.isort]
+multi_line_output=3
+force_grid_wrap=1
+include_trailing_comma = true
+line_length = 120
+combine_as_imports = true
+lines_after_imports = 2
+extra_standard_library = ["dataclasses"]
+known_project = ["explicit", "testapp"]
+sections = ["FUTURE","STDLIB","THIRDPARTY","FIRSTPARTY","PROJECT","LOCALFOLDER"]
+
+
+[tool.ruff]
+target-version = "py312"
+line-length = 120
+indent-width = 4
+
+
+[tool.ruff.lint]
+select = ["F", "E", "D", "W", "S", "N",]
+ignore = [
+    # Missing docstring in public module
+    "D100",
+    # Missing docstring in public package
+    "D104",
+    # Missing docstring in __init__
+    "D107",
+    # use of assert detected (useless with pytest)
+    "S101",
+]
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".idea",
+    ".ipynb_checkpoints",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pyenv",
+    ".pytest_cache",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    ".vscode",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "site-packages",
+    "venv",
+    "py-libs/**",
+]
+
+[tool.ruff.format]
+quote-style = "single"
+indent-style = "space"
+docstring-code-format = true
+
+[tool.ruff.lint.per-file-ignores]
+"*/tests/*" = ["D"]
+"*/migrations/*" = ["D"]
+
+[tool.mypy]
+plugins = ["pydantic.mypy"]
+no_site_packages = true
+ignore_missing_imports = true
+exclude = "migrations/"

explicit_python_dlq-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

explicit_python_dlq-1.0.0/src/explicit/dlq/__init__.py

@@ -0,0 +1 @@
+

explicit_python_dlq-1.0.0/src/explicit/dlq/adapters/db.py

@@ -0,0 +1,13 @@
+"""Адаптеры для работы с Dead Letter Queue (DLQ) в базе данных."""
+
+from abc import (
+    ABCMeta,
+)
+
+from explicit.adapters.db import (
+    AbstractRepository,
+)
+
+
+class AbstractDLQRepository(AbstractRepository, metaclass=ABCMeta):
+    """Абстрактный репозиторий для работы с DLQ."""

explicit_python_dlq-1.0.0/src/explicit/dlq/config.py

@@ -0,0 +1,63 @@
+"""Конфигурация DLQ."""
+
+from dataclasses import (
+    dataclass,
+)
+from typing import (
+    TYPE_CHECKING,
+)
+
+from pydantic import (
+    ConfigDict,
+)
+
+from explicit.dlq.domain import (
+    commands,
+)
+from explicit.dlq.services.handlers import (
+    DLQHandlers,
+)
+
+
+if TYPE_CHECKING:
+    from explicit.dlq.adapters.db import (
+        AbstractDLQRepository,
+    )
+    from explicit.dlq.types import (
+        DLQMessageDispatcher,
+    )
+    from explicit.messagebus import (
+        MessageBus,
+    )
+
+
+@dataclass
+class DLQConfig:
+    """Конфигурация DLQ."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    bus: 'MessageBus'
+    repository: 'AbstractDLQRepository'
+    message_dispatcher: 'DLQMessageDispatcher'
+
+
+def configure_dlq(config: DLQConfig):
+    """Настроить DLQ."""
+    config.bus.get_uow().register_repositories(('dead_letters', config.repository))
+
+    handlers = DLQHandlers(bus=config.bus, message_dispatcher=config.message_dispatcher)
+
+    config.bus.add_command_handler(
+        commands.RegisterDeadLetter,
+        handlers.register_dead_letter,  # type: ignore[attr-defined,arg-type]
+    )
+    config.bus.add_command_handler(
+        commands.ProcessDeadLetter,
+        handlers.process_dead_letter,  # type: ignore[attr-defined,arg-type]
+    )
+
+    config.bus.add_command_handler(
+        commands.UpdateDeadLetterRawMessage,
+        handlers.update_raw_message_value,  # type: ignore[attr-defined,arg-type]
+    )

explicit_python_dlq-1.0.0/src/explicit/dlq/contrib/django/README.md

@@ -0,0 +1,40 @@
+## Набор компонентов для интеграции explicit.dlq с Django.
+Содержит реализацию репозитория необработанных сообщений.
+
+## Пример подключения
+testapp/settings.py:
+```python
+INSTALLED_APPS = [
+    # другие приложения
+    'explicit.dlq.contrib.django',  # подключение приложения с моделью DeadLetter
+    'testapp.core',                 # настройка компонентов из explicit.dlq
+]
+```
+
+testapp/dlq/apps.py:
+```python
+
+from datetime import timedelta
+
+from django.apps import AppConfig as AppConfigBase
+
+
+class AppConfig(AppConfigBase):
+    name = __package__
+
+    def ready(self):
+        self._configure_dlq()
+
+    def _configure_dlq(self):
+        from explicit.dlq.config import DLQConfig, configure_dlq
+        
+        # реализация репозитория на базе Django ORM 
+        from explicit.dlq.contrib.django.adapters.db import Repository
+        config = DLQConfig(
+            # ...
+            repository=Repository(),
+            # ...
+        )
+        # регистрация репозитория и регистрация обработчиков команд
+        configure_dlq(config)
+```

explicit_python_dlq-1.0.0/src/explicit/dlq/contrib/django/adapters/db.py

@@ -0,0 +1,104 @@
+"""Адаптеры для работы с Dead Letter Queue (DLQ) в базе данных."""
+
+from typing import (
+    TYPE_CHECKING,
+    Iterator,
+    Union,
+)
+
+from django.core.exceptions import (
+    ObjectDoesNotExist,
+)
+
+from explicit.dlq.adapters.db import (
+    AbstractDLQRepository,
+)
+from explicit.dlq.domain.model import (
+    DeadLetter,
+    DeadLetterNotFound,
+)
+from explicit.domain import (
+    asdict,
+)
+
+
+if TYPE_CHECKING:
+    from uuid import (
+        UUID,
+    )
+
+    from explicit.dlq.contrib.django import (
+        models as db,
+    )
+
+
+class Repository(AbstractDLQRepository):
+    """Репозиторий DeadLetterQueue."""
+
+    @property
+    def _base_qs(self):
+        # pylint: disable-next=import-outside-toplevel
+        from explicit.dlq.contrib.django.models import (
+            DeadLetter as DBDeadLetter,
+        )
+
+        return DBDeadLetter.objects.order_by('_first_attempted_at')
+
+    def _to_domain(self, dbinstance: 'db.DeadLetter') -> DeadLetter:
+        return DeadLetter(
+            id=dbinstance.id,
+            raw_message_value=bytes(dbinstance.raw_message_value),
+            raw_message_key=bytes(dbinstance.raw_message_key) if dbinstance.raw_message_key else None,
+            topic=dbinstance.topic,
+            attempts=tuple(dbinstance.attempts),
+            processed_at=dbinstance.processed_at,
+        )
+
+    def _to_db(self, modelinstance: DeadLetter) -> DeadLetter:
+        # pylint: disable-next=import-outside-toplevel
+        from explicit.dlq.contrib.django.models import (
+            DeadLetter as DBDeadLetter,
+        )
+
+        assert isinstance(modelinstance, DeadLetter)
+        db_instance, _ = DBDeadLetter.objects.update_or_create(
+            pk=modelinstance.id,
+            defaults=asdict(modelinstance, exclude={'id', 'attempts'})
+            | {
+                'attempts': [
+                    asdict(attempt) | {'failed_at': attempt.failed_at.isoformat()} for attempt in modelinstance.attempts
+                ],
+                '_first_attempted_at': modelinstance.attempts[0].failed_at,
+                '_attempts_count': len(modelinstance.attempts),
+            },
+        )
+        return self.get_object_by_id(db_instance.pk)
+
+    def add(self, obj: DeadLetter) -> DeadLetter:
+        """Добавить сообщение."""
+        return self._to_db(obj)
+
+    def update(self, obj: DeadLetter) -> DeadLetter:
+        """Обновить сообщение."""
+        assert isinstance(obj, DeadLetter)
+
+        return self._to_db(obj)
+
+    def delete(self, obj: DeadLetter) -> None:
+        """Удалить сообщение."""
+        assert isinstance(obj, DeadLetter)
+
+        self._base_qs.filter(pk=obj.id).delete()
+
+    def get_all_objects(self) -> Iterator[DeadLetter]:
+        """Получить все сообщения."""
+        for db_instance in self._base_qs.iterator():
+            yield self._to_domain(db_instance)
+
+    def get_object_by_id(self, identifier: 'Union[UUID, str]') -> DeadLetter:
+        """Получить сообщение по идентификатору."""
+        try:
+            db_instance = self._base_qs.get(pk=identifier)
+            return self._to_domain(db_instance)
+        except ObjectDoesNotExist as exc:
+            raise DeadLetterNotFound from exc

explicit_python_dlq-1.0.0/src/explicit/dlq/contrib/django/apps.py

@@ -0,0 +1,12 @@
+"""Конфигурация приложения."""
+
+from django.apps import (
+    AppConfig as AppConfigBase,
+)
+
+
+class AppConfig(AppConfigBase):
+    """Конфигурация приложения."""
+
+    name = __package__
+    label = 'explicit_dlq_django'

explicit_python_dlq-1.0.0/src/explicit/dlq/contrib/django/migrations/0001_initial.py

@@ -0,0 +1,60 @@
+# pylint: disable=missing-module-docstring,invalid-name
+import uuid
+
+from django import (
+    VERSION as DJANGO_VERSION,
+)
+from django.db import (
+    migrations,
+    models,
+)
+
+
+if DJANGO_VERSION >= (4, 0):
+    from django.db.models import (
+        JSONField,
+    )
+else:
+    from django.contrib.postgres.fields import (
+        JSONField,
+    )
+
+
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name='DeadLetter',
+            fields=[
+                (
+                    'id',
+                    models.UUIDField(
+                        default=uuid.uuid4,
+                        editable=False,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='Идентификатор UUID',
+                    ),
+                ),
+                ('raw_message_value', models.BinaryField(verbose_name='Содержимое сообщения')),
+                ('raw_message_key', models.BinaryField(verbose_name='Ключ сообщения', null=True, blank=True)),
+                ('topic', models.CharField(max_length=256, verbose_name='Топик сообщения')),
+                ('attempts', JSONField(verbose_name='Попытки обработки сообщения')),
+                (
+                    'processed_at',
+                    models.DateTimeField(blank=True, null=True, verbose_name='Время успешной обработки сообщения'),
+                ),
+                ('_first_attempted_at', models.DateTimeField(db_index=True)),
+                ('_attempts_count', models.IntegerField(db_index=True)),
+            ],
+            options={
+                'verbose_name': 'Необработанное сообщение',
+                'verbose_name_plural': 'Необработанные сообщения',
+                'indexes': [models.Index(fields=['_attempts_count', '_first_attempted_at'], name='dlq_attempts_idx')],
+                'db_table': 'dlq_dead_letter',
+            },
+        ),
+    ]