django-streaming 0.4__tar.gz → 0.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. {django_streaming-0.4 → django_streaming-0.5}/PKG-INFO +2 -1
  2. {django_streaming-0.4 → django_streaming-0.5}/pyproject.toml +2 -0
  3. django_streaming-0.5/src/streaming/__cli__.py +198 -0
  4. {django_streaming-0.4 → django_streaming-0.5}/src/streaming/backends/__init__.py +4 -1
  5. django_streaming-0.5/src/streaming/backends/_base.py +87 -0
  6. django_streaming-0.5/src/streaming/backends/console.py +30 -0
  7. django_streaming-0.5/src/streaming/backends/debug.py +28 -0
  8. django_streaming-0.5/src/streaming/backends/rabbitmq.py +225 -0
  9. {django_streaming-0.4 → django_streaming-0.5}/src/streaming/config.py +13 -7
  10. {django_streaming-0.4 → django_streaming-0.5}/src/streaming/exceptions.py +4 -0
  11. {django_streaming-0.4 → django_streaming-0.5}/src/streaming/manager.py +15 -10
  12. django_streaming-0.5/src/streaming/py.typed +0 -0
  13. {django_streaming-0.4 → django_streaming-0.5}/src/streaming/types.pyi +7 -3
  14. django_streaming-0.5/src/streaming/utils.py +78 -0
  15. {django_streaming-0.4 → django_streaming-0.5}/src/streaming/version.py +2 -2
  16. django_streaming-0.4/src/streaming/__cli__.py +0 -158
  17. django_streaming-0.4/src/streaming/backends/_base.py +0 -26
  18. django_streaming-0.4/src/streaming/backends/console.py +0 -23
  19. django_streaming-0.4/src/streaming/backends/debug.py +0 -18
  20. django_streaming-0.4/src/streaming/backends/rabbitmq.py +0 -200
  21. django_streaming-0.4/src/streaming/handlers.py +0 -3
  22. django_streaming-0.4/src/streaming/utils.py +0 -30
  23. {django_streaming-0.4 → django_streaming-0.5}/.gitignore +0 -0
  24. {django_streaming-0.4 → django_streaming-0.5}/LICENSE +0 -0
  25. {django_streaming-0.4 → django_streaming-0.5}/README.md +0 -0
  26. /django_streaming-0.4/src/streaming/api.py → /django_streaming-0.5/src/streaming/__init__.py +0 -0
  27. /django_streaming-0.4/src/streaming/py.typed → /django_streaming-0.5/src/streaming/api.py +0 -0
  28. {django_streaming-0.4 → django_streaming-0.5}/src/streaming/apps.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: django-streaming
3
- Version: 0.4
3
+ Version: 0.5
4
4
  Summary: A project for streaming data.
5
5
  Project-URL: Documentation, https://github.com/UNICEF/django-streaming
6
6
  Project-URL: Homepage, https://github.com/UNICEF/django-streaming
@@ -23,6 +23,7 @@ Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
23
23
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
24
24
  Requires-Python: >=3.10
25
25
  Requires-Dist: click>=8
26
+ Requires-Dist: colorama>=0.4.6
26
27
  Requires-Dist: django-stubs-ext
27
28
  Requires-Dist: django>=3
28
29
  Requires-Dist: markdown>=3.8.2
@@ -35,6 +35,7 @@ dynamic = [
35
35
  ]
36
36
  dependencies = [
37
37
  "click>=8",
38
+ "colorama>=0.4.6",
38
39
  "django>=3",
39
40
  "django-stubs-ext",
40
41
  "markdown>=3.8.2",
@@ -52,6 +53,7 @@ dev = [
52
53
  "django-webtest>=1.9.13",
53
54
  "factory-boy>=3.3.3",
54
55
  "pre-commit>=3.3.3",
56
+ "pyrabbit2>=1.0.7",
55
57
  "pytest>=8.4.1",
56
58
  "pytest-cov>=6.2.1",
57
59
  "pytest-django>=4.11.1",
@@ -0,0 +1,198 @@
1
+ import json
2
+ import logging
3
+ from typing import TYPE_CHECKING
4
+
5
+ import click
6
+ from click import ClickException
7
+ from colorama import Fore, Style
8
+ from django.core.exceptions import ImproperlyConfigured
9
+ from pika.adapters.blocking_connection import BlockingChannel
10
+ from pika.exceptions import ChannelClosedByBroker
11
+ from pika.spec import Basic, BasicProperties
12
+
13
+ from .backends import RabbitMQBackend, get_backend
14
+ from .exceptions import AuthorizationError, StreamingConfigError
15
+ from .utils import make_event
16
+
17
+ if TYPE_CHECKING:
18
+ from .types import JSON, EventType
19
+
20
+
21
+ logger = logging.getLogger(__name__)
22
+
23
+
24
+ def _dump_info(backend: "RabbitMQBackend") -> None:
25
+ from streaming.config import CONFIG
26
+
27
+ line = f"{Fore.YELLOW}%-16s: {Style.RESET_ALL}%s"
28
+ click.secho(line % ("Server", f"{backend.host}:{backend.port}"))
29
+ click.secho(line % ("VirtualHost", f"{backend.virtual_host}"))
30
+ click.secho(line % ("Exchange", f"{backend.exchange}"))
31
+ click.secho(line % ("Queues", ""))
32
+ for alias, config in CONFIG.QUEUES.items():
33
+ click.secho(line % (f" {alias}", f"{config}"))
34
+ click.secho(line % ("Timeout", f"{backend.timeout}"))
35
+ click.secho(line % ("Client Name", f"{backend.client_name}"))
36
+
37
+
38
+ def assert_backend() -> "RabbitMQBackend":
39
+ backend: RabbitMQBackend = get_backend() # type: ignore[assignment]
40
+
41
+ if not isinstance(backend, RabbitMQBackend):
42
+ raise ClickException("RabbitMQ backend is not configured")
43
+ return backend
44
+
45
+
46
+ @click.group()
47
+ def cli() -> None:
48
+ """Streaming CLI."""
49
+ try:
50
+ import django
51
+
52
+ django.setup()
53
+ except ModuleNotFoundError as e:
54
+ raise ClickException(f"Unable to setup Django. {e}") from e
55
+ except ImproperlyConfigured as e:
56
+ raise ClickException("Unable to setup Django. Is DJANGO_SETTINGS_MODULE environment variable set?") from e
57
+
58
+
59
+ @cli.command()
60
+ @click.option("--client-name", default=None, help="Override client name")
61
+ def configure(client_name: str) -> None:
62
+ from streaming.config import CONFIG
63
+
64
+ backend: RabbitMQBackend = assert_backend()
65
+ if client_name:
66
+ backend.client_name = client_name
67
+ try:
68
+ backend.connect(True)
69
+ backend.configure_exchanges()
70
+ backend.configure_queue_routing()
71
+ check.callback() # type: ignore[misc]
72
+ except AuthorizationError as e:
73
+ click.secho(f"Unable to connect using {CONFIG.BROKER_URL}", fg="red", err=True)
74
+ raise ClickException(str(e)) from e
75
+ except StreamingConfigError as e:
76
+ click.secho(f"Generic error {e}", fg="red", err=True)
77
+ raise ClickException(str(e)) from e
78
+
79
+
80
+ @cli.command()
81
+ @click.argument("routing_key")
82
+ @click.option("-c", "--client-name", default=None, help="Override client name")
83
+ @click.option("--message", default="Test Message", help="Message to send")
84
+ @click.option("--debug", is_flag=True, help="Debug mode")
85
+ def send(routing_key: str, message: str, client_name: str, debug: bool) -> None:
86
+ backend = assert_backend()
87
+ logger = logging.getLogger("streaming")
88
+ if debug:
89
+ logger.setLevel(logging.DEBUG)
90
+ logger.addHandler(logging.StreamHandler())
91
+ else:
92
+ logger.handlers = []
93
+
94
+ if client_name:
95
+ backend.client_name = client_name
96
+ try:
97
+ payload = json.loads(message)
98
+ except json.decoder.JSONDecodeError:
99
+ payload = {
100
+ "message": message,
101
+ }
102
+ msg: EventType = make_event(payload, event="Test")
103
+ backend.publish(routing_key, msg)
104
+ click.secho(f"Sent: {msg}")
105
+ backend.disconnect()
106
+
107
+
108
+ def _listen(queues: list[str], payload: bool, pretty: bool, client_name: str) -> None:
109
+ backend = assert_backend()
110
+
111
+ if client_name:
112
+ backend.client_name = client_name
113
+
114
+ def callback(
115
+ queue_name: str, ch: BlockingChannel, method: Basic.Deliver, properties: BasicProperties, body: bytes
116
+ ) -> None:
117
+ message: EventType = json.loads(body.decode())
118
+ click.echo(
119
+ f"{Fore.GREEN}{message['timestamp']} "
120
+ f"[{queue_name}]"
121
+ f"{Fore.LIGHTWHITE_EX} [{message['type']}]"
122
+ f"{message['event']} "
123
+ )
124
+ extra: str | JSON
125
+ if payload:
126
+ if pretty:
127
+ extra = json.dumps(message["payload"], indent=4)
128
+ else:
129
+ extra = message["payload"]
130
+ click.echo(f"{Fore.YELLOW}{extra}{Fore.RESET}")
131
+
132
+ try:
133
+ backend.connect()
134
+ _dump_info(backend)
135
+ backend.listen(callback, queues=queues)
136
+ except KeyboardInterrupt:
137
+ click.secho("Stopping listener.", fg="yellow")
138
+ finally:
139
+ backend.disconnect()
140
+
141
+
142
+ @cli.command()
143
+ @click.option("-q", "--queues", multiple=True, help="Queue name to listen to")
144
+ @click.option("-c", "--client-name", default=None, help="Override client name")
145
+ @click.option("--payload", default=False, is_flag=True, help="Print payload")
146
+ @click.option("--autoreload", "reload", is_flag=True, help="Enable auto-reloading.")
147
+ @click.option("--pretty", is_flag=True, help="Pretty-print payload.")
148
+ def listen(queues: list[str], payload: bool, reload: bool, pretty: bool, client_name: str) -> None:
149
+ """Listens for streaming events."""
150
+ if reload:
151
+ from django.utils import autoreload
152
+
153
+ click.secho("Starting listener with autoreload...", fg="yellow")
154
+ autoreload.run_with_reloader(_listen, queues=queues, payload=payload, pretty=pretty, client_name=client_name)
155
+ else:
156
+ _listen(queues=queues, payload=payload, pretty=pretty, client_name=client_name)
157
+
158
+
159
+ @cli.command()
160
+ def purge() -> None:
161
+ """Purges all messages from the configured queues."""
162
+ from streaming.backends.rabbitmq import RabbitMQBackend
163
+ from streaming.config import CONFIG
164
+ from streaming.manager import initialize_engine
165
+
166
+ manager = initialize_engine(True)
167
+ backend = manager.backend
168
+ if not isinstance(backend, RabbitMQBackend):
169
+ raise click.ClickException("RabbitMQ backend is not configured. Please set BROKER_URL to a rabbit:// URL.")
170
+
171
+ backend.connect(True)
172
+ for queue_alias, queue_config in CONFIG.QUEUES.items():
173
+ queue_name = queue_config.get("name", queue_alias)
174
+ try:
175
+ message_count = backend.channel.queue_purge(queue_name) # type: ignore[union-attr]
176
+ click.secho(f"Purged {message_count.method.message_count} messages from queue '{queue_name}'.", fg="green")
177
+ except ChannelClosedByBroker:
178
+ click.secho(f"Could not purge queue '{queue_name}'. Queue may not exist.", fg="red")
179
+ backend.disconnect()
180
+
181
+
182
+ @cli.command()
183
+ def check() -> None:
184
+ """Checks the streaming configuration and connection."""
185
+ from streaming.config import CONFIG
186
+
187
+ click.secho("System Configuration:")
188
+ config_dict = dict(CONFIG._parsed)
189
+ for key, value in config_dict.items():
190
+ click.echo(f" {key}: {value}")
191
+
192
+ backend: RabbitMQBackend = assert_backend()
193
+ try:
194
+ backend.connect(True)
195
+ click.secho("Connection successful.", fg="green")
196
+ _dump_info(backend)
197
+ except (StreamingConfigError, AuthorizationError) as e:
198
+ raise ClickException(f"Connection failed: {e}") from e
@@ -2,13 +2,16 @@ import logging
2
2
  from typing import TYPE_CHECKING
3
3
  from urllib.parse import urlparse
4
4
 
5
- from streaming.exceptions import StreamingConfigError
5
+ from ..exceptions import StreamingConfigError
6
+ from .rabbitmq import RabbitMQBackend
6
7
 
7
8
  if TYPE_CHECKING:
8
9
  from ._base import BaseBackend
9
10
 
10
11
  logger = logging.getLogger(__name__)
11
12
 
13
+ __all__ = ["RabbitMQBackend", "get_backend"]
14
+
12
15
 
13
16
  def get_backend() -> "BaseBackend":
14
17
  from streaming.config import CONFIG
@@ -0,0 +1,87 @@
1
+ import atexit
2
+ import logging
3
+ from abc import ABC, abstractmethod
4
+ from typing import TYPE_CHECKING, Any
5
+ from urllib.parse import ParseResult, parse_qs, urlparse
6
+
7
+ from streaming.config import CONFIG
8
+ from streaming.utils import get_local_ip
9
+
10
+ from ..exceptions import (
11
+ StreamingError,
12
+ )
13
+
14
+ if TYPE_CHECKING:
15
+ from pika.adapters.blocking_connection import BlockingChannel, BlockingConnection
16
+
17
+ from streaming.types import EventType, UserCallback
18
+
19
+
20
+ logger = logging.getLogger(__name__)
21
+
22
+ MAX_RETRIES = 5
23
+
24
+
25
+ class BaseBackend(ABC):
26
+ def __init__(self, url: str) -> None:
27
+ self.connection_url: str = url
28
+ self._parsed_url: ParseResult = urlparse(self.connection_url)
29
+ self._options = {k: v[0] for k, v in parse_qs(self._parsed_url.query).items()}
30
+ self.exchange = self.get_option("exchange", "django-streaming-broadcast")
31
+
32
+ self.host = str(self._parsed_url.hostname)
33
+ self.port = int(self._parsed_url.port) if self._parsed_url.port else 5672
34
+ self._connection: BlockingConnection | None = None
35
+ self._username = self._parsed_url.username or "guest"
36
+ self._password = self._parsed_url.password or "guest"
37
+
38
+ self.channel: BlockingChannel | None = None
39
+ self.exchange = self.get_option("exchange", "django-streaming-broadcast")
40
+ self.retry_exchange = f"retry_{self.exchange}"
41
+ self.timeout = float(self.get_option("timeout", 0.5))
42
+ self.virtual_host = self.get_option("vhost", "/")
43
+ # listener
44
+ self.client_name = CONFIG.CLIENT_NAME or get_local_ip()
45
+ atexit.register(self.disconnect)
46
+
47
+ def get_option(self, name: str, default: Any = "") -> Any:
48
+ return self._options.get(name, default)
49
+
50
+ @property
51
+ def client_name(self) -> str:
52
+ return self.__client_name
53
+
54
+ @client_name.setter
55
+ def client_name(self, name: str) -> None:
56
+ self.__client_name = name
57
+
58
+ def configure_exchanges(self) -> None: # noqa: B027
59
+ pass
60
+
61
+ def configure_queue_routing(self) -> None: # noqa: B027
62
+ pass
63
+
64
+ def get_real_queue_name(self, name: str) -> str:
65
+ return f"{self.client_name}:{name}"
66
+
67
+ def set_credential(self, username: str, password: str) -> None:
68
+ if self._connection:
69
+ raise StreamingError("Disconnect first.")
70
+ self.__username = username
71
+ self.__password = password
72
+
73
+ @abstractmethod
74
+ def connect(self, raise_if_error: bool = False) -> None:
75
+ pass
76
+
77
+ @abstractmethod
78
+ def disconnect(self) -> None:
79
+ pass
80
+
81
+ # Publisher
82
+ @abstractmethod
83
+ def publish(self, routing_key: str, message: "EventType") -> bool: ...
84
+
85
+ @abstractmethod
86
+ def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
87
+ pass
@@ -0,0 +1,30 @@
1
+ import logging
2
+ import sys
3
+ from typing import TYPE_CHECKING
4
+
5
+ from ._base import BaseBackend
6
+
7
+ logger = logging.getLogger(__name__)
8
+
9
+ if TYPE_CHECKING:
10
+ from ..types import EventType, UserCallback
11
+
12
+
13
+ class ConsoleBackend(BaseBackend):
14
+ def __init__(self, url: str) -> None:
15
+ super().__init__(url)
16
+ self.stream = self.get_option("stream", "stdout")
17
+
18
+ def publish(self, routing_key: str, message: "EventType") -> bool:
19
+ stream = getattr(sys, self.stream)
20
+ stream.write(f"routing_key:{routing_key} message:{message}\n")
21
+ return True
22
+
23
+ def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
24
+ pass
25
+
26
+ def connect(self, raise_if_error: bool = False) -> None:
27
+ pass
28
+
29
+ def disconnect(self) -> None:
30
+ pass
@@ -0,0 +1,28 @@
1
+ import logging
2
+ from typing import TYPE_CHECKING
3
+
4
+ from ._base import BaseBackend
5
+
6
+ logger = logging.getLogger(__name__)
7
+
8
+ if TYPE_CHECKING:
9
+ from ..types import EventType, UserCallback
10
+
11
+
12
+ class DebugBackend(BaseBackend):
13
+ def __init__(self, url: str) -> None:
14
+ super().__init__(url)
15
+ self.messages: list[tuple[str, EventType]] = []
16
+
17
+ def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
18
+ pass
19
+
20
+ def connect(self, raise_if_error: bool = False) -> None:
21
+ pass
22
+
23
+ def disconnect(self) -> None:
24
+ pass
25
+
26
+ def publish(self, routing_key: str, message: "EventType") -> bool:
27
+ self.messages.append((routing_key, message))
28
+ return True
@@ -0,0 +1,225 @@
1
+ import json
2
+ import logging
3
+ import socket
4
+ import time
5
+ from typing import TYPE_CHECKING
6
+
7
+ import pika.channel
8
+ import pika.exceptions
9
+ from pika import PlainCredentials
10
+ from pika.exceptions import ConnectionClosedByBroker, ConnectionWrongStateError
11
+ from pika.exchange_type import ExchangeType
12
+
13
+ from streaming.config import CONFIG
14
+ from streaming.utils import exchange_exists
15
+
16
+ from ..exceptions import (
17
+ AuthorizationError,
18
+ StreamingCallbackError,
19
+ StreamingCallbackFailure,
20
+ StreamingConfigError,
21
+ )
22
+ from ..utils import json_dumps
23
+ from ._base import BaseBackend
24
+
25
+ if TYPE_CHECKING:
26
+ from pika.adapters.blocking_connection import BlockingChannel
27
+ from pika.spec import Basic, BasicProperties
28
+
29
+ from streaming.types import EventType, UserCallback
30
+
31
+ logger = logging.getLogger(__name__)
32
+
33
+ MAX_RETRIES = 5
34
+
35
+
36
+ class Callback:
37
+ def __init__(
38
+ self, queue_name: str, backend: "RabbitMQBackend", user_callback: "UserCallback", ack: bool = True
39
+ ) -> None:
40
+ self.backend = backend
41
+ self.user_callback: UserCallback = user_callback
42
+ self.ack = ack
43
+ self.queue_name = queue_name
44
+
45
+ def __call__(
46
+ self, ch: "BlockingChannel", method: "Basic.Deliver", properties: "BasicProperties", body: bytes
47
+ ) -> None:
48
+ try:
49
+ self.user_callback(self.queue_name, ch, method, properties, body)
50
+ if self.ack:
51
+ ch.basic_ack(delivery_tag=method.delivery_tag) # type: ignore[arg-type]
52
+ except StreamingCallbackError as e:
53
+ evt: EventType = json.loads(body.decode())
54
+ retries = int(properties.headers.get("x-retries", 0)) # type: ignore[union-attr]
55
+ ch.basic_ack(method.delivery_tag) # type: ignore[arg-type]
56
+ self.backend._handle_retry(evt, ch, method, retries)
57
+ logger.debug("StreamingCallbackError", exc_info=e)
58
+ except StreamingCallbackFailure as e:
59
+ logger.error(f"Callback failure: {e}", exc_info=e)
60
+ except Exception as e:
61
+ logger.exception(f"Unexpected exception occurred: {e}", exc_info=e)
62
+
63
+
64
+ class RabbitMQBackend(BaseBackend):
65
+ def __init__(self, url: str) -> None:
66
+ super().__init__(url)
67
+
68
+ def check_channel(self) -> None:
69
+ if not self.channel:
70
+ raise StreamingConfigError("No active channel")
71
+
72
+ def configure_exchanges(self) -> None:
73
+ logger.debug("Configuring exchanges")
74
+ if not self.channel:
75
+ raise StreamingConfigError("No active channel")
76
+ unrouted_exchange = f"{self.exchange}_unrouted"
77
+ dead_letter_queue = f"{self.exchange}_unrouted_queue"
78
+ logger.debug(f"Declaring exchange '{unrouted_exchange}'")
79
+ self.channel.exchange_declare(exchange=unrouted_exchange, exchange_type=ExchangeType.fanout, durable=True)
80
+ self.channel.queue_declare(queue=dead_letter_queue, durable=True)
81
+ self.channel.queue_bind(exchange=unrouted_exchange, queue=dead_letter_queue)
82
+
83
+ logger.debug(f"Declaring exchange '{self.exchange}'")
84
+ self.channel.exchange_declare(
85
+ exchange=self.exchange,
86
+ exchange_type=ExchangeType.topic,
87
+ durable=True,
88
+ arguments={"alternate-exchange": unrouted_exchange},
89
+ )
90
+ self.channel.exchange_declare(self.retry_exchange, exchange_type=ExchangeType.direct, durable=True)
91
+
92
+ def configure_queue_routing(self) -> None:
93
+ self.check_channel()
94
+ if not exchange_exists(self.channel, self.exchange):
95
+ raise StreamingConfigError("Exchange not found")
96
+
97
+ for alias, config in CONFIG.QUEUES.items():
98
+ real_name = self.get_real_queue_name(alias)
99
+ logger.debug(f"Declaring queue '{real_name}'")
100
+ self.channel.queue_declare(queue=real_name, durable=True, arguments=None) # type: ignore[union-attr]
101
+ binding_keys = config.get("routing", [])
102
+ for binding_key in binding_keys:
103
+ logger.debug("Listening on queue '%s' routed by '%s'", alias, binding_key)
104
+ self.channel.queue_bind(exchange=self.exchange, queue=real_name, routing_key=binding_key) # type: ignore[union-attr]
105
+
106
+ def connect(self, raise_if_error: bool = False) -> None:
107
+ logger.debug("Connecting to %s:%s", self.host, self.port)
108
+ if self._connection and self._connection.is_open:
109
+ self.disconnect()
110
+
111
+ for __ in range(CONFIG.RETRY_COUNT):
112
+ try:
113
+ auth = PlainCredentials(self._username, self._password)
114
+ self._connection = pika.BlockingConnection(
115
+ pika.ConnectionParameters(
116
+ host=self.host,
117
+ port=self.port,
118
+ virtual_host=self.virtual_host,
119
+ credentials=auth,
120
+ socket_timeout=self.timeout,
121
+ blocked_connection_timeout=self.timeout,
122
+ stack_timeout=self.timeout,
123
+ client_properties={
124
+ "connection_name": self.client_name,
125
+ "product": "django-streaming",
126
+ "information": "",
127
+ "version": "1.0",
128
+ },
129
+ )
130
+ )
131
+ self.channel = self._connection.channel()
132
+ return
133
+ except (
134
+ pika.exceptions.AuthenticationError,
135
+ pika.exceptions.ProbableAuthenticationError,
136
+ pika.exceptions.ProbableAccessDeniedError,
137
+ ) as e:
138
+ raise AuthorizationError(str(e)) from e
139
+ except (socket.gaierror, pika.exceptions.AMQPError) as e:
140
+ logger.warning(
141
+ f"Could not connect to RabbitMQ. Retrying in {CONFIG.RETRY_DELAY} seconds...",
142
+ )
143
+ time.sleep(CONFIG.RETRY_DELAY)
144
+ if raise_if_error:
145
+ raise StreamingConfigError(f"Error connecting {self.connection_url}") from e
146
+ logger.critical("Could not connect to RabbitMQ after multiple retries.")
147
+
148
+ def disconnect(self) -> None:
149
+ try:
150
+ if self._connection:
151
+ logger.debug("Closing RabbitMQ connection.")
152
+ self._connection.close()
153
+ except (ConnectionClosedByBroker, AttributeError, ConnectionWrongStateError):
154
+ pass
155
+ finally:
156
+ self._connection = None
157
+ self.channel = None
158
+
159
+ # Publisher
160
+ def publish(self, routing_key: str, message: "EventType") -> bool:
161
+ try:
162
+ if not self.channel or self.channel.is_closed:
163
+ self.connect(True)
164
+ logger.debug(f"Publishing to exchange '{self.exchange}' using routing key '{routing_key}'")
165
+ self._basic_publish(message, routing_key, 0)
166
+ return True
167
+ except Exception as e: # noqa: BLE001
168
+ logger.critical("Unhandled error sending to RabbitMQ. Message not published.", exc_info=e)
169
+ return False
170
+
171
+ # Listener
172
+
173
+ def _basic_publish(self, message: "EventType", routing_key: str, retry_count: int = 0) -> None:
174
+ self.check_channel()
175
+ self.channel.basic_publish( # type: ignore[union-attr]
176
+ exchange=self.exchange,
177
+ routing_key=routing_key,
178
+ body=json_dumps(message).encode(),
179
+ properties=pika.BasicProperties(
180
+ delivery_mode=pika.spec.PERSISTENT_DELIVERY_MODE,
181
+ expiration=str(CONFIG.MESSAGE_TTL * 1000), # milliseconds
182
+ headers={"x-retries": retry_count},
183
+ ),
184
+ )
185
+
186
+ def _handle_retry(self, message: "EventType", ch: "BlockingChannel", method: "Basic.Deliver", retries: int) -> None:
187
+ ch.basic_ack(method.delivery_tag) # type: ignore[arg-type]
188
+ if retries < MAX_RETRIES:
189
+ delay = 2000 * (2**retries) # ms (exponential backoff)
190
+ delay_queue = f"{self.exchange}_retry_{delay}ms"
191
+
192
+ # Declare a delay queue with TTL
193
+ ch.queue_declare(
194
+ delay_queue,
195
+ durable=True,
196
+ arguments={
197
+ "x-dead-letter-exchange": self.exchange, # after delay → back to main
198
+ "x-message-ttl": delay,
199
+ },
200
+ )
201
+ ch.queue_bind(delay_queue, self.retry_exchange, routing_key="task")
202
+ self._basic_publish(message, routing_key=method.routing_key or "*", retry_count=retries + 1)
203
+ else:
204
+ logger.error(f"Dropping after {MAX_RETRIES} retries")
205
+
206
+ def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
207
+ if self.channel is None:
208
+ self.connect()
209
+ self.check_channel()
210
+
211
+ configured_queues = CONFIG.QUEUES
212
+ if not configured_queues:
213
+ logger.warning("No queues configured in settings.STREAMING['QUEUES']")
214
+ return
215
+ queues_to_listen = queues or configured_queues.keys()
216
+ for queue_name in queues_to_listen:
217
+ if queue_name not in configured_queues:
218
+ logger.error(f"Queue '{queue_name}' not found in configured listening queues. Ignored.")
219
+ continue
220
+ real_queue_name = self.get_real_queue_name(queue_name)
221
+ _callback = Callback(queue_name, self, callback, ack=ack)
222
+ self.channel.basic_consume(queue=real_queue_name, on_message_callback=_callback, auto_ack=False) # type: ignore[union-attr]
223
+
224
+ logger.info("Waiting for messages. To exit press CTRL+C")
225
+ self.channel.start_consuming() # type: ignore[union-attr]
@@ -8,21 +8,23 @@ from django.dispatch import receiver
8
8
 
9
9
  logger = logging.getLogger(__name__)
10
10
 
11
- DEFAULT_QUEUE_NAME = "django_model_changes"
12
-
13
11
 
14
12
  class StreamingConfig:
15
13
  _DEFAULTS = {
16
14
  "BROKER_URL": "console://",
15
+ "QUEUES": {},
16
+ "CLIENT_NAME": "",
17
17
  "RETRY_COUNT": 3,
18
18
  "RETRY_DELAY": 1,
19
+ "MESSAGE_TTL": 60 * 60 * 24 * 2, # 2 days
19
20
  "MANAGER_CLASS": "streaming.manager.ChangeManager",
20
21
  }
21
22
 
22
23
  def __init__(self) -> None:
23
- self._overrides = getattr(settings, "STREAMING", {})
24
- self._parsed = ChainMap(self._overrides, self._DEFAULTS)
25
- self._cached: dict[str, Any] = {}
24
+ self._overrides: dict[str, Any]
25
+ self._parsed: ChainMap[str, Any]
26
+ self._cached: dict[str, Any]
27
+ self.load()
26
28
 
27
29
  def __getattr__(self, name: str) -> Any:
28
30
  if name not in self._parsed:
@@ -31,12 +33,16 @@ class StreamingConfig:
31
33
  self._cached[name] = self._parsed[name]
32
34
  return self._cached[name]
33
35
 
36
+ def load(self) -> None:
37
+ self._overrides = getattr(settings, "STREAMING", {})
38
+ self._parsed = ChainMap(self._overrides, self._DEFAULTS)
39
+ self._cached = {}
40
+
34
41
 
35
42
  CONFIG = StreamingConfig()
36
43
 
37
44
 
38
45
  @receiver(setting_changed)
39
46
  def reload_config(sender: Any, setting: str, **kwargs: Any) -> None:
40
- global CONFIG # noqa: PLW0603
41
47
  if setting == "STREAMING":
42
- CONFIG = StreamingConfig()
48
+ CONFIG.load()
@@ -11,6 +11,10 @@ class StreamingConfigError(StreamingError):
11
11
  pass
12
12
 
13
13
 
14
+ class AuthorizationError(StreamingConfigError):
15
+ pass
16
+
17
+
14
18
  class StreamingBackendError(StreamingError):
15
19
  pass
16
20