django-streaming 0.4__tar.gz → 0.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. {django_streaming-0.4 → django_streaming-0.6.0}/PKG-INFO +2 -1
  2. {django_streaming-0.4 → django_streaming-0.6.0}/pyproject.toml +2 -0
  3. django_streaming-0.6.0/src/streaming/__cli__.py +202 -0
  4. {django_streaming-0.4 → django_streaming-0.6.0}/src/streaming/backends/__init__.py +4 -1
  5. django_streaming-0.6.0/src/streaming/backends/_base.py +90 -0
  6. django_streaming-0.6.0/src/streaming/backends/console.py +37 -0
  7. django_streaming-0.6.0/src/streaming/backends/debug.py +35 -0
  8. django_streaming-0.6.0/src/streaming/backends/rabbitmq.py +232 -0
  9. {django_streaming-0.4 → django_streaming-0.6.0}/src/streaming/config.py +13 -7
  10. django_streaming-0.6.0/src/streaming/event.py +64 -0
  11. {django_streaming-0.4 → django_streaming-0.6.0}/src/streaming/exceptions.py +5 -1
  12. {django_streaming-0.4 → django_streaming-0.6.0}/src/streaming/manager.py +16 -11
  13. {django_streaming-0.4 → django_streaming-0.6.0}/src/streaming/types.pyi +5 -7
  14. django_streaming-0.6.0/src/streaming/utils.py +50 -0
  15. {django_streaming-0.4 → django_streaming-0.6.0}/src/streaming/version.py +2 -2
  16. django_streaming-0.4/src/streaming/__cli__.py +0 -158
  17. django_streaming-0.4/src/streaming/backends/_base.py +0 -26
  18. django_streaming-0.4/src/streaming/backends/console.py +0 -23
  19. django_streaming-0.4/src/streaming/backends/debug.py +0 -18
  20. django_streaming-0.4/src/streaming/backends/rabbitmq.py +0 -200
  21. django_streaming-0.4/src/streaming/handlers.py +0 -3
  22. django_streaming-0.4/src/streaming/utils.py +0 -30
  23. {django_streaming-0.4 → django_streaming-0.6.0}/.gitignore +0 -0
  24. {django_streaming-0.4 → django_streaming-0.6.0}/LICENSE +0 -0
  25. {django_streaming-0.4 → django_streaming-0.6.0}/README.md +0 -0
  26. /django_streaming-0.4/src/streaming/api.py → /django_streaming-0.6.0/src/streaming/__init__.py +0 -0
  27. {django_streaming-0.4 → django_streaming-0.6.0}/src/streaming/apps.py +0 -0
  28. {django_streaming-0.4 → django_streaming-0.6.0}/src/streaming/py.typed +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: django-streaming
3
- Version: 0.4
3
+ Version: 0.6.0
4
4
  Summary: A project for streaming data.
5
5
  Project-URL: Documentation, https://github.com/UNICEF/django-streaming
6
6
  Project-URL: Homepage, https://github.com/UNICEF/django-streaming
@@ -23,6 +23,7 @@ Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
23
23
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
24
24
  Requires-Python: >=3.10
25
25
  Requires-Dist: click>=8
26
+ Requires-Dist: colorama>=0.4.6
26
27
  Requires-Dist: django-stubs-ext
27
28
  Requires-Dist: django>=3
28
29
  Requires-Dist: markdown>=3.8.2
@@ -35,6 +35,7 @@ dynamic = [
35
35
  ]
36
36
  dependencies = [
37
37
  "click>=8",
38
+ "colorama>=0.4.6",
38
39
  "django>=3",
39
40
  "django-stubs-ext",
40
41
  "markdown>=3.8.2",
@@ -52,6 +53,7 @@ dev = [
52
53
  "django-webtest>=1.9.13",
53
54
  "factory-boy>=3.3.3",
54
55
  "pre-commit>=3.3.3",
56
+ "pyrabbit2>=1.0.7",
55
57
  "pytest>=8.4.1",
56
58
  "pytest-cov>=6.2.1",
57
59
  "pytest-django>=4.11.1",
@@ -0,0 +1,202 @@
1
+ import json
2
+ import logging
3
+ from typing import TYPE_CHECKING
4
+
5
+ import click
6
+ from click import ClickException
7
+ from colorama import Fore, Style
8
+ from django.core.exceptions import ImproperlyConfigured
9
+ from pika.adapters.blocking_connection import BlockingChannel
10
+ from pika.exceptions import ChannelClosedByBroker
11
+ from pika.spec import Basic, BasicProperties
12
+
13
+ from .backends import RabbitMQBackend, get_backend
14
+ from .event import Event
15
+ from .exceptions import AuthorizationError, StreamingConfigError
16
+ from .utils import make_event
17
+
18
+ if TYPE_CHECKING:
19
+ from .types import JSON
20
+
21
+ logger = logging.getLogger(__name__)
22
+ LINE = f"{Fore.YELLOW}%-16s: {Style.RESET_ALL}%s"
23
+
24
+
25
+ def _dump_info(backend: "RabbitMQBackend") -> None:
26
+ click.secho(LINE % ("Server", f"{backend.host}:{backend.port}"))
27
+ click.secho(LINE % ("VirtualHost", f"{backend.virtual_host}"))
28
+ click.secho(LINE % ("Exchange", f"{backend.exchange}"))
29
+ click.secho(LINE % ("Timeout", f"{backend.timeout}"))
30
+ click.secho(LINE % ("Client Name", f"{backend.client_name}"))
31
+
32
+
33
+ def assert_backend() -> "RabbitMQBackend":
34
+ backend: RabbitMQBackend = get_backend() # type: ignore[assignment]
35
+
36
+ if not isinstance(backend, RabbitMQBackend):
37
+ raise ClickException("RabbitMQ backend is not configured")
38
+ return backend
39
+
40
+
41
+ def configure_logging(debug: bool, loggers: tuple[str, ...] = ("streaming",)) -> None:
42
+ for log_name in loggers:
43
+ logr = logging.getLogger(log_name)
44
+ if debug:
45
+ logr.setLevel(logging.DEBUG)
46
+ logr.addHandler(logging.StreamHandler())
47
+ else:
48
+ logr.handlers = []
49
+
50
+
51
+ @click.group()
52
+ def cli() -> None:
53
+ """Streaming CLI."""
54
+ try:
55
+ import django
56
+
57
+ django.setup()
58
+ except ModuleNotFoundError as e:
59
+ raise ClickException(f"Unable to setup Django. {e}") from e
60
+ except ImproperlyConfigured as e:
61
+ raise ClickException("Unable to setup Django. Is DJANGO_SETTINGS_MODULE environment variable set?") from e
62
+
63
+
64
+ @cli.command()
65
+ @click.option("--queues/--no-queues", "queues", is_flag=True, default=False, help="Debug mode")
66
+ @click.option("--debug", is_flag=True, help="Debug mode")
67
+ def configure(queues: bool = False, debug: bool = False) -> None:
68
+ from streaming.config import CONFIG
69
+
70
+ routing = None
71
+ configure_logging(debug)
72
+ backend: RabbitMQBackend = assert_backend()
73
+ try:
74
+ backend.connect(True)
75
+ backend.configure_exchanges()
76
+ if queues:
77
+ routing = backend.configure_queue_routing()
78
+ check.callback() # type: ignore[misc]
79
+ if routing:
80
+ click.secho(LINE % ("Queues", ""))
81
+ for k, v in routing.items():
82
+ click.secho(LINE % (f" {k}", "; ".join(v)))
83
+ except AuthorizationError as e:
84
+ click.secho(f"Unable to connect using {CONFIG.BROKER_URL}", fg="red", err=True)
85
+ raise ClickException(str(e)) from e
86
+ except StreamingConfigError as e:
87
+ click.secho(f"Generic error {e}", fg="red", err=True)
88
+ raise ClickException(str(e)) from e
89
+
90
+
91
+ @cli.command()
92
+ @click.argument("routing_key")
93
+ @click.option("-c", "--client-name", default=None, help="Override client name")
94
+ @click.option("--message", default="Test Message", help="Message to send")
95
+ @click.option("--debug", is_flag=True, help="Debug mode")
96
+ def send(routing_key: str, message: str, client_name: str, debug: bool) -> None:
97
+ backend = assert_backend()
98
+ configure_logging(debug)
99
+
100
+ if client_name:
101
+ backend.client_name = client_name
102
+ try:
103
+ payload = json.loads(message)
104
+ except json.decoder.JSONDecodeError:
105
+ payload = {
106
+ "message": message,
107
+ }
108
+ msg: Event = make_event(payload, key="Test")
109
+ backend.publish(routing_key, msg)
110
+ click.secho(f"Sent: {msg}")
111
+ backend.disconnect()
112
+
113
+
114
+ def _listen(queues: list[str], payload: bool, pretty: bool) -> None:
115
+ backend = assert_backend()
116
+
117
+ def callback(
118
+ queue_name: str, ch: BlockingChannel, method: Basic.Deliver, properties: BasicProperties, body: bytes
119
+ ) -> None:
120
+ message: Event = Event.unmarshal(body)
121
+ click.echo(f"{Fore.GREEN}{message.timestamp} [{queue_name}]{Fore.LIGHTWHITE_EX} [{message.key}] {message.id} ")
122
+ extra: str | JSON
123
+ if payload:
124
+ if pretty:
125
+ extra = json.dumps(message.payload, indent=4)
126
+ else:
127
+ extra = message.payload
128
+ click.echo(f"{Fore.YELLOW}{extra}{Fore.RESET}")
129
+
130
+ try:
131
+ backend.connect(True)
132
+ _dump_info(backend)
133
+ backend.listen(callback, queues=queues)
134
+ except ChannelClosedByBroker as e:
135
+ click.secho(str(e), fg="red")
136
+ if "no queue" in str(e):
137
+ click.secho("Did you run 'stream configure --queues'", fg="red")
138
+ except StreamingConfigError as e:
139
+ click.secho(str(e), fg="red")
140
+ click.get_current_context().exit(2)
141
+ except KeyboardInterrupt:
142
+ click.secho("Stopping listener.", fg="yellow")
143
+ finally:
144
+ backend.disconnect()
145
+
146
+
147
+ @cli.command()
148
+ @click.option("-q", "--queues", multiple=True, help="Queue name to listen to")
149
+ @click.option("--payload", default=False, is_flag=True, help="Print payload")
150
+ @click.option("--autoreload", "reload", is_flag=True, help="Enable auto-reloading.")
151
+ @click.option("--pretty", is_flag=True, help="Pretty-print payload.")
152
+ def listen(queues: list[str], payload: bool, reload: bool, pretty: bool) -> None:
153
+ """Listens for streaming events."""
154
+ if reload:
155
+ from django.utils import autoreload
156
+
157
+ click.secho("Starting listener with autoreload...", fg="yellow")
158
+ autoreload.run_with_reloader(_listen, queues=queues, payload=payload, pretty=pretty)
159
+ else:
160
+ _listen(queues=queues, payload=payload, pretty=pretty)
161
+
162
+
163
+ @cli.command()
164
+ def purge() -> None:
165
+ """Purges all messages from the configured queues."""
166
+ from streaming.backends.rabbitmq import RabbitMQBackend
167
+ from streaming.config import CONFIG
168
+ from streaming.manager import initialize_engine
169
+
170
+ manager = initialize_engine(True)
171
+ backend = manager.backend
172
+ if not isinstance(backend, RabbitMQBackend):
173
+ raise click.ClickException("RabbitMQ backend is not configured. Please set BROKER_URL to a rabbit:// URL.")
174
+
175
+ backend.connect(True)
176
+ for queue_alias, queue_config in CONFIG.QUEUES.items():
177
+ queue_name = queue_config.get("name", queue_alias)
178
+ try:
179
+ message_count = backend.channel.queue_purge(queue_name) # type: ignore[union-attr]
180
+ click.secho(f"Purged {message_count.method.message_count} messages from queue '{queue_name}'.", fg="green")
181
+ except ChannelClosedByBroker:
182
+ click.secho(f"Could not purge queue '{queue_name}'. Queue may not exist.", fg="red")
183
+ backend.disconnect()
184
+
185
+
186
+ @cli.command()
187
+ def check() -> None:
188
+ """Checks the streaming configuration and connection."""
189
+ from streaming.config import CONFIG
190
+
191
+ click.secho("System Configuration:")
192
+ config_dict = dict(CONFIG._parsed)
193
+ for key, value in config_dict.items():
194
+ click.echo(f" {key}: {value}")
195
+
196
+ backend: RabbitMQBackend = assert_backend()
197
+ try:
198
+ backend.connect(True)
199
+ click.secho("Connection successful.", fg="green")
200
+ _dump_info(backend)
201
+ except (StreamingConfigError, AuthorizationError) as e:
202
+ raise ClickException(f"Connection failed: {e}") from e
@@ -2,13 +2,16 @@ import logging
2
2
  from typing import TYPE_CHECKING
3
3
  from urllib.parse import urlparse
4
4
 
5
- from streaming.exceptions import StreamingConfigError
5
+ from ..exceptions import StreamingConfigError
6
+ from .rabbitmq import RabbitMQBackend
6
7
 
7
8
  if TYPE_CHECKING:
8
9
  from ._base import BaseBackend
9
10
 
10
11
  logger = logging.getLogger(__name__)
11
12
 
13
+ __all__ = ["RabbitMQBackend", "get_backend"]
14
+
12
15
 
13
16
  def get_backend() -> "BaseBackend":
14
17
  from streaming.config import CONFIG
@@ -0,0 +1,90 @@
1
+ import atexit
2
+ import logging
3
+ from abc import ABC, abstractmethod
4
+ from typing import TYPE_CHECKING, Any
5
+ from urllib.parse import ParseResult, parse_qs, urlparse
6
+
7
+ from streaming.config import CONFIG
8
+ from streaming.utils import get_local_ip
9
+
10
+ from ..exceptions import (
11
+ StreamingError,
12
+ )
13
+
14
+ if TYPE_CHECKING:
15
+ from pika.adapters.blocking_connection import BlockingChannel, BlockingConnection
16
+
17
+ from ..event import Event
18
+ from ..types import UserCallback
19
+
20
+
21
+ logger = logging.getLogger(__name__)
22
+
23
+ MAX_RETRIES = 5
24
+
25
+
26
+ class BaseBackend(ABC):
27
+ def __init__(self, url: str) -> None:
28
+ self.connection_url: str = url
29
+ self._parsed_url: ParseResult = urlparse(self.connection_url)
30
+ self._options = {k: v[0] for k, v in parse_qs(self._parsed_url.query).items()}
31
+ self.exchange = self.get_option("exchange", "django-streaming-broadcast")
32
+
33
+ self.host = str(self._parsed_url.hostname)
34
+ self.port = int(self._parsed_url.port) if self._parsed_url.port else 5672
35
+ self._connection: BlockingConnection | None = None
36
+ self._username = self._parsed_url.username or "guest"
37
+ self._password = self._parsed_url.password or "guest"
38
+
39
+ self.channel: BlockingChannel | None = None
40
+ self.exchange = self.get_option("exchange", "django-streaming-broadcast")
41
+ self.retry_exchange = f"retry_{self.exchange}"
42
+ self.timeout = float(self.get_option("timeout", 0.5))
43
+ self.virtual_host = self.get_option("vhost", "/")
44
+ # listener
45
+ self.client_name = CONFIG.CLIENT_NAME or self.get_option("client_name", get_local_ip())
46
+ atexit.register(self.disconnect)
47
+
48
+ def get_option(self, name: str, default: Any = "") -> Any:
49
+ return self._options.get(name, default)
50
+
51
+ @property
52
+ def client_name(self) -> str:
53
+ return self.__client_name
54
+
55
+ @client_name.setter
56
+ def client_name(self, name: str) -> None:
57
+ self.__client_name = name
58
+
59
+ @abstractmethod
60
+ def configure_exchanges(self) -> None: # noqa: B027
61
+ ...
62
+
63
+ @abstractmethod
64
+ def configure_queue_routing(self) -> dict[str, list[str]]: # noqa: B027
65
+ ...
66
+
67
+ def get_real_queue_name(self, name: str) -> str:
68
+ return f"{self.client_name}:{name}"
69
+
70
+ def set_credential(self, username: str, password: str) -> None:
71
+ if self._connection:
72
+ raise StreamingError("Disconnect first.")
73
+ self.__username = username
74
+ self.__password = password
75
+
76
+ @abstractmethod
77
+ def connect(self, raise_if_error: bool = False) -> None:
78
+ pass
79
+
80
+ @abstractmethod
81
+ def disconnect(self) -> None:
82
+ pass
83
+
84
+ # Publisher
85
+ @abstractmethod
86
+ def publish(self, routing_key: str, message: "Event") -> bool: ...
87
+
88
+ @abstractmethod
89
+ def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
90
+ pass
@@ -0,0 +1,37 @@
1
+ import logging
2
+ import sys
3
+ from typing import TYPE_CHECKING
4
+
5
+ from ._base import BaseBackend
6
+
7
+ logger = logging.getLogger(__name__)
8
+
9
+ if TYPE_CHECKING:
10
+ from ..event import Event
11
+ from ..types import UserCallback
12
+
13
+
14
+ class ConsoleBackend(BaseBackend):
15
+ def __init__(self, url: str) -> None:
16
+ super().__init__(url)
17
+ self.stream = self.get_option("stream", "stdout")
18
+
19
+ def configure_queue_routing(self) -> dict[str, list[str]]: # pragma: no cover
20
+ return {}
21
+
22
+ def configure_exchanges(self) -> None: # pragma: no cover
23
+ pass
24
+
25
+ def publish(self, routing_key: str, message: "Event") -> bool:
26
+ stream = getattr(sys, self.stream)
27
+ stream.write(f"routing_key:{routing_key} message:{message}\n")
28
+ return True
29
+
30
+ def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
31
+ pass
32
+
33
+ def connect(self, raise_if_error: bool = False) -> None:
34
+ pass
35
+
36
+ def disconnect(self) -> None:
37
+ pass
@@ -0,0 +1,35 @@
1
+ import logging
2
+ from typing import TYPE_CHECKING
3
+
4
+ from ._base import BaseBackend
5
+
6
+ logger = logging.getLogger(__name__)
7
+
8
+ if TYPE_CHECKING:
9
+ from ..event import Event
10
+ from ..types import UserCallback
11
+
12
+
13
+ class DebugBackend(BaseBackend):
14
+ def configure_queue_routing(self) -> dict[str, list[str]]: # pragma: no cover
15
+ return {}
16
+
17
+ def configure_exchanges(self) -> None: # pragma: no cover
18
+ pass
19
+
20
+ def __init__(self, url: str) -> None:
21
+ super().__init__(url)
22
+ self.messages: list[tuple[str, Event]] = []
23
+
24
+ def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
25
+ pass
26
+
27
+ def connect(self, raise_if_error: bool = False) -> None:
28
+ pass
29
+
30
+ def disconnect(self) -> None:
31
+ pass
32
+
33
+ def publish(self, routing_key: str, message: "Event") -> bool:
34
+ self.messages.append((routing_key, message))
35
+ return True
@@ -0,0 +1,232 @@
1
+ import logging
2
+ import socket
3
+ import time
4
+ from collections import defaultdict
5
+ from contextlib import suppress
6
+ from typing import TYPE_CHECKING
7
+
8
+ import pika.channel
9
+ import pika.exceptions
10
+ from pika import PlainCredentials
11
+ from pika.exceptions import ConnectionClosedByBroker, ConnectionWrongStateError
12
+ from pika.exchange_type import ExchangeType
13
+
14
+ from streaming.config import CONFIG
15
+ from streaming.utils import exchange_exists
16
+
17
+ from ..event import Event
18
+ from ..exceptions import (
19
+ AuthorizationError,
20
+ StreamingCallbackFailure,
21
+ StreamingCallbackRetryError,
22
+ StreamingConfigError,
23
+ )
24
+ from ._base import BaseBackend
25
+
26
+ if TYPE_CHECKING:
27
+ from pika.adapters.blocking_connection import BlockingChannel
28
+ from pika.spec import Basic, BasicProperties
29
+
30
+ from streaming.types import UserCallback
31
+
32
+ logger = logging.getLogger(__name__)
33
+
34
+ MAX_RETRIES = 5
35
+
36
+
37
+ class Callback:
38
+ def __init__(
39
+ self, queue_name: str, backend: "RabbitMQBackend", user_callback: "UserCallback", ack: bool = True
40
+ ) -> None:
41
+ self.backend = backend
42
+ self.user_callback: UserCallback = user_callback
43
+ self.ack = ack
44
+ self.queue_name = queue_name
45
+
46
+ def __call__(
47
+ self, ch: "BlockingChannel", method: "Basic.Deliver", properties: "BasicProperties", body: bytes
48
+ ) -> None:
49
+ try:
50
+ self.user_callback(self.queue_name, ch, method, properties, body)
51
+ if self.ack:
52
+ ch.basic_ack(delivery_tag=method.delivery_tag) # type: ignore[arg-type]
53
+ except StreamingCallbackRetryError as e:
54
+ evt: Event = Event.unmarshal(body)
55
+ logger.debug(f"StreamingCallbackError {evt.id}", exc_info=e)
56
+ retries = int(properties.headers.get("x-retries", 0)) # type: ignore[union-attr]
57
+ self.backend._handle_retry(evt, ch, method, retries)
58
+ except StreamingCallbackFailure as e:
59
+ logger.error(f"Callback failure: {e}", exc_info=e)
60
+ except Exception as e:
61
+ logger.exception(f"Unexpected exception occurred: {e}", exc_info=e)
62
+
63
+
64
+ class RabbitMQBackend(BaseBackend):
65
+ def __init__(self, url: str) -> None:
66
+ super().__init__(url)
67
+
68
+ def check_channel(self) -> None:
69
+ if not self.channel:
70
+ raise StreamingConfigError("No active channel")
71
+
72
+ def configure_exchanges(self) -> None:
73
+ logger.debug("Configuring exchanges")
74
+ if not self.channel:
75
+ raise StreamingConfigError("No active channel")
76
+ unrouted_exchange = f"{self.exchange}_unrouted"
77
+ dead_letter_queue = f"{self.exchange}_unrouted_queue"
78
+ logger.debug(f"Declaring exchange '{unrouted_exchange}'")
79
+ self.channel.exchange_declare(exchange=unrouted_exchange, exchange_type=ExchangeType.fanout, durable=True)
80
+ self.channel.queue_declare(queue=dead_letter_queue, durable=True)
81
+ self.channel.queue_bind(exchange=unrouted_exchange, queue=dead_letter_queue)
82
+
83
+ logger.debug(f"Declaring exchange '{self.exchange}'")
84
+ self.channel.exchange_declare(
85
+ exchange=self.exchange,
86
+ exchange_type=ExchangeType.topic,
87
+ durable=True,
88
+ arguments={"alternate-exchange": unrouted_exchange},
89
+ )
90
+ self.channel.exchange_declare(self.retry_exchange, exchange_type=ExchangeType.direct, durable=True)
91
+
92
+ def configure_queue_routing(self) -> dict[str, list[str]]:
93
+ self.check_channel()
94
+ if not exchange_exists(self.channel, self.exchange):
95
+ raise StreamingConfigError("Exchange not found")
96
+ ret = defaultdict(list)
97
+ for alias, config in CONFIG.QUEUES.items():
98
+ real_name = self.get_real_queue_name(alias)
99
+ logger.debug(f"Declaring queue '{real_name}'")
100
+ self.channel.queue_declare(queue=real_name, durable=True, arguments=None) # type: ignore[union-attr]
101
+ if binding_keys := config.get("routing", []):
102
+ for binding_key in binding_keys:
103
+ logger.debug("Listening on queue '%s' routed by '%s'", alias, binding_key)
104
+ self.channel.queue_bind(exchange=self.exchange, queue=real_name, routing_key=binding_key) # type: ignore[union-attr]
105
+ ret[alias].append(binding_key)
106
+ else:
107
+ ret[alias].append("warning no routing defined")
108
+ return ret
109
+
110
+ def connect(self, raise_if_error: bool = False) -> None:
111
+ logger.debug("Connecting to %s:%s", self.host, self.port)
112
+ if self._connection and self._connection.is_open:
113
+ self.disconnect()
114
+
115
+ for __ in range(CONFIG.RETRY_COUNT):
116
+ try:
117
+ auth = PlainCredentials(self._username, self._password)
118
+ self._connection = pika.BlockingConnection(
119
+ pika.ConnectionParameters(
120
+ host=self.host,
121
+ port=self.port,
122
+ virtual_host=self.virtual_host,
123
+ credentials=auth,
124
+ socket_timeout=self.timeout,
125
+ blocked_connection_timeout=self.timeout,
126
+ stack_timeout=self.timeout,
127
+ client_properties={
128
+ "connection_name": self.client_name,
129
+ "product": "django-streaming",
130
+ "information": "",
131
+ "version": "1.0",
132
+ },
133
+ )
134
+ )
135
+ self.channel = self._connection.channel()
136
+ return
137
+ except (
138
+ pika.exceptions.AuthenticationError,
139
+ pika.exceptions.ProbableAuthenticationError,
140
+ pika.exceptions.ProbableAccessDeniedError,
141
+ ) as e:
142
+ raise AuthorizationError(str(e)) from e
143
+ except (socket.gaierror, pika.exceptions.AMQPError) as e:
144
+ logger.warning(
145
+ f"Could not connect to RabbitMQ. Retrying in {CONFIG.RETRY_DELAY} seconds...",
146
+ )
147
+ time.sleep(CONFIG.RETRY_DELAY)
148
+ if raise_if_error:
149
+ raise StreamingConfigError(f"Error connecting {self.connection_url}") from e
150
+ logger.critical("Could not connect to RabbitMQ after multiple retries.")
151
+
152
+ def disconnect(self) -> None:
153
+ try:
154
+ if self._connection:
155
+ with suppress(ConnectionClosedByBroker, AttributeError, ConnectionWrongStateError):
156
+ logger.debug("Closing RabbitMQ connection.")
157
+ self._connection.close()
158
+ finally:
159
+ self._connection = None
160
+ self.channel = None
161
+
162
+ # Publisher
163
+ def publish(self, routing_key: str, message: "Event") -> bool:
164
+ try:
165
+ if not self.channel or self.channel.is_closed:
166
+ self.connect(True)
167
+ logger.debug(f"Publishing to exchange '{self.exchange}' using routing key '{routing_key}'")
168
+ self._basic_publish(message, routing_key, 0)
169
+ return True
170
+ except Exception as e: # noqa: BLE001
171
+ logger.critical("Unhandled error sending to RabbitMQ. Message not published.", exc_info=e)
172
+ return False
173
+
174
+ # Listener
175
+
176
+ def _basic_publish(
177
+ self, message: "Event", routing_key: str, retry_count: int = 0, exchange: str | None = None
178
+ ) -> None:
179
+ self.check_channel()
180
+ self.channel.basic_publish( # type: ignore[union-attr]
181
+ exchange=exchange or self.exchange,
182
+ routing_key=routing_key,
183
+ body=message.marshall(),
184
+ properties=pika.BasicProperties(
185
+ delivery_mode=pika.spec.PERSISTENT_DELIVERY_MODE,
186
+ expiration=str(CONFIG.MESSAGE_TTL * 1000), # milliseconds
187
+ headers={"x-retries": retry_count},
188
+ ),
189
+ )
190
+
191
+ def _handle_retry(self, message: "Event", ch: "BlockingChannel", method: "Basic.Deliver", retries: int) -> None:
192
+ ch.basic_ack(method.delivery_tag) # type: ignore[arg-type]
193
+ if retries < MAX_RETRIES:
194
+ delay = 2000 * (2**retries) # ms (exponential backoff)
195
+ delay_queue = f"{self.exchange}_retry_{method.routing_key}_{delay}ms"
196
+ logger.debug(
197
+ f"Retrying message {message.id} ({retries + 1}/{MAX_RETRIES}) "
198
+ f"with routing key '{method.routing_key}' in {delay / 1000}s"
199
+ )
200
+
201
+ # Declare a delay queue with TTL
202
+ ch.queue_declare(
203
+ delay_queue,
204
+ durable=True,
205
+ arguments={
206
+ "x-dead-letter-exchange": self.exchange, # after delay → back to main
207
+ "x-message-ttl": delay,
208
+ "x-dead-letter-routing-key": method.routing_key,
209
+ },
210
+ )
211
+ ch.queue_bind(delay_queue, self.retry_exchange, routing_key=delay_queue)
212
+ self._basic_publish(message, routing_key=delay_queue, retry_count=retries + 1, exchange=self.retry_exchange)
213
+ else:
214
+ logger.error(f"Dropping message {message.id} after {MAX_RETRIES} retries")
215
+
216
+ def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
217
+ if self.channel is None:
218
+ self.connect()
219
+ self.check_channel()
220
+
221
+ configured_queues = CONFIG.QUEUES
222
+ if not configured_queues:
223
+ logger.warning("No queues configured in settings.STREAMING['QUEUES']")
224
+ return
225
+ queues_to_listen = queues or configured_queues.keys()
226
+ for queue_name in queues_to_listen:
227
+ real_queue_name = self.get_real_queue_name(queue_name)
228
+ _callback = Callback(queue_name, self, callback, ack=ack)
229
+ self.channel.basic_consume(queue=real_queue_name, on_message_callback=_callback, auto_ack=False) # type: ignore[union-attr]
230
+
231
+ logger.info("Waiting for messages. To exit press CTRL+C")
232
+ self.channel.start_consuming() # type: ignore[union-attr]