django-streaming 0.4__tar.gz → 0.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {django_streaming-0.4 → django_streaming-0.5}/PKG-INFO +2 -1
- {django_streaming-0.4 → django_streaming-0.5}/pyproject.toml +2 -0
- django_streaming-0.5/src/streaming/__cli__.py +198 -0
- {django_streaming-0.4 → django_streaming-0.5}/src/streaming/backends/__init__.py +4 -1
- django_streaming-0.5/src/streaming/backends/_base.py +87 -0
- django_streaming-0.5/src/streaming/backends/console.py +30 -0
- django_streaming-0.5/src/streaming/backends/debug.py +28 -0
- django_streaming-0.5/src/streaming/backends/rabbitmq.py +225 -0
- {django_streaming-0.4 → django_streaming-0.5}/src/streaming/config.py +13 -7
- {django_streaming-0.4 → django_streaming-0.5}/src/streaming/exceptions.py +4 -0
- {django_streaming-0.4 → django_streaming-0.5}/src/streaming/manager.py +15 -10
- django_streaming-0.5/src/streaming/py.typed +0 -0
- {django_streaming-0.4 → django_streaming-0.5}/src/streaming/types.pyi +7 -3
- django_streaming-0.5/src/streaming/utils.py +78 -0
- {django_streaming-0.4 → django_streaming-0.5}/src/streaming/version.py +2 -2
- django_streaming-0.4/src/streaming/__cli__.py +0 -158
- django_streaming-0.4/src/streaming/backends/_base.py +0 -26
- django_streaming-0.4/src/streaming/backends/console.py +0 -23
- django_streaming-0.4/src/streaming/backends/debug.py +0 -18
- django_streaming-0.4/src/streaming/backends/rabbitmq.py +0 -200
- django_streaming-0.4/src/streaming/handlers.py +0 -3
- django_streaming-0.4/src/streaming/utils.py +0 -30
- {django_streaming-0.4 → django_streaming-0.5}/.gitignore +0 -0
- {django_streaming-0.4 → django_streaming-0.5}/LICENSE +0 -0
- {django_streaming-0.4 → django_streaming-0.5}/README.md +0 -0
- /django_streaming-0.4/src/streaming/api.py → /django_streaming-0.5/src/streaming/__init__.py +0 -0
- /django_streaming-0.4/src/streaming/py.typed → /django_streaming-0.5/src/streaming/api.py +0 -0
- {django_streaming-0.4 → django_streaming-0.5}/src/streaming/apps.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: django-streaming
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5
|
|
4
4
|
Summary: A project for streaming data.
|
|
5
5
|
Project-URL: Documentation, https://github.com/UNICEF/django-streaming
|
|
6
6
|
Project-URL: Homepage, https://github.com/UNICEF/django-streaming
|
|
@@ -23,6 +23,7 @@ Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
|
|
|
23
23
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
24
24
|
Requires-Python: >=3.10
|
|
25
25
|
Requires-Dist: click>=8
|
|
26
|
+
Requires-Dist: colorama>=0.4.6
|
|
26
27
|
Requires-Dist: django-stubs-ext
|
|
27
28
|
Requires-Dist: django>=3
|
|
28
29
|
Requires-Dist: markdown>=3.8.2
|
|
@@ -35,6 +35,7 @@ dynamic = [
|
|
|
35
35
|
]
|
|
36
36
|
dependencies = [
|
|
37
37
|
"click>=8",
|
|
38
|
+
"colorama>=0.4.6",
|
|
38
39
|
"django>=3",
|
|
39
40
|
"django-stubs-ext",
|
|
40
41
|
"markdown>=3.8.2",
|
|
@@ -52,6 +53,7 @@ dev = [
|
|
|
52
53
|
"django-webtest>=1.9.13",
|
|
53
54
|
"factory-boy>=3.3.3",
|
|
54
55
|
"pre-commit>=3.3.3",
|
|
56
|
+
"pyrabbit2>=1.0.7",
|
|
55
57
|
"pytest>=8.4.1",
|
|
56
58
|
"pytest-cov>=6.2.1",
|
|
57
59
|
"pytest-django>=4.11.1",
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
|
+
|
|
5
|
+
import click
|
|
6
|
+
from click import ClickException
|
|
7
|
+
from colorama import Fore, Style
|
|
8
|
+
from django.core.exceptions import ImproperlyConfigured
|
|
9
|
+
from pika.adapters.blocking_connection import BlockingChannel
|
|
10
|
+
from pika.exceptions import ChannelClosedByBroker
|
|
11
|
+
from pika.spec import Basic, BasicProperties
|
|
12
|
+
|
|
13
|
+
from .backends import RabbitMQBackend, get_backend
|
|
14
|
+
from .exceptions import AuthorizationError, StreamingConfigError
|
|
15
|
+
from .utils import make_event
|
|
16
|
+
|
|
17
|
+
if TYPE_CHECKING:
|
|
18
|
+
from .types import JSON, EventType
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger(__name__)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _dump_info(backend: "RabbitMQBackend") -> None:
|
|
25
|
+
from streaming.config import CONFIG
|
|
26
|
+
|
|
27
|
+
line = f"{Fore.YELLOW}%-16s: {Style.RESET_ALL}%s"
|
|
28
|
+
click.secho(line % ("Server", f"{backend.host}:{backend.port}"))
|
|
29
|
+
click.secho(line % ("VirtualHost", f"{backend.virtual_host}"))
|
|
30
|
+
click.secho(line % ("Exchange", f"{backend.exchange}"))
|
|
31
|
+
click.secho(line % ("Queues", ""))
|
|
32
|
+
for alias, config in CONFIG.QUEUES.items():
|
|
33
|
+
click.secho(line % (f" {alias}", f"{config}"))
|
|
34
|
+
click.secho(line % ("Timeout", f"{backend.timeout}"))
|
|
35
|
+
click.secho(line % ("Client Name", f"{backend.client_name}"))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def assert_backend() -> "RabbitMQBackend":
|
|
39
|
+
backend: RabbitMQBackend = get_backend() # type: ignore[assignment]
|
|
40
|
+
|
|
41
|
+
if not isinstance(backend, RabbitMQBackend):
|
|
42
|
+
raise ClickException("RabbitMQ backend is not configured")
|
|
43
|
+
return backend
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@click.group()
|
|
47
|
+
def cli() -> None:
|
|
48
|
+
"""Streaming CLI."""
|
|
49
|
+
try:
|
|
50
|
+
import django
|
|
51
|
+
|
|
52
|
+
django.setup()
|
|
53
|
+
except ModuleNotFoundError as e:
|
|
54
|
+
raise ClickException(f"Unable to setup Django. {e}") from e
|
|
55
|
+
except ImproperlyConfigured as e:
|
|
56
|
+
raise ClickException("Unable to setup Django. Is DJANGO_SETTINGS_MODULE environment variable set?") from e
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@cli.command()
|
|
60
|
+
@click.option("--client-name", default=None, help="Override client name")
|
|
61
|
+
def configure(client_name: str) -> None:
|
|
62
|
+
from streaming.config import CONFIG
|
|
63
|
+
|
|
64
|
+
backend: RabbitMQBackend = assert_backend()
|
|
65
|
+
if client_name:
|
|
66
|
+
backend.client_name = client_name
|
|
67
|
+
try:
|
|
68
|
+
backend.connect(True)
|
|
69
|
+
backend.configure_exchanges()
|
|
70
|
+
backend.configure_queue_routing()
|
|
71
|
+
check.callback() # type: ignore[misc]
|
|
72
|
+
except AuthorizationError as e:
|
|
73
|
+
click.secho(f"Unable to connect using {CONFIG.BROKER_URL}", fg="red", err=True)
|
|
74
|
+
raise ClickException(str(e)) from e
|
|
75
|
+
except StreamingConfigError as e:
|
|
76
|
+
click.secho(f"Generic error {e}", fg="red", err=True)
|
|
77
|
+
raise ClickException(str(e)) from e
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@cli.command()
|
|
81
|
+
@click.argument("routing_key")
|
|
82
|
+
@click.option("-c", "--client-name", default=None, help="Override client name")
|
|
83
|
+
@click.option("--message", default="Test Message", help="Message to send")
|
|
84
|
+
@click.option("--debug", is_flag=True, help="Debug mode")
|
|
85
|
+
def send(routing_key: str, message: str, client_name: str, debug: bool) -> None:
|
|
86
|
+
backend = assert_backend()
|
|
87
|
+
logger = logging.getLogger("streaming")
|
|
88
|
+
if debug:
|
|
89
|
+
logger.setLevel(logging.DEBUG)
|
|
90
|
+
logger.addHandler(logging.StreamHandler())
|
|
91
|
+
else:
|
|
92
|
+
logger.handlers = []
|
|
93
|
+
|
|
94
|
+
if client_name:
|
|
95
|
+
backend.client_name = client_name
|
|
96
|
+
try:
|
|
97
|
+
payload = json.loads(message)
|
|
98
|
+
except json.decoder.JSONDecodeError:
|
|
99
|
+
payload = {
|
|
100
|
+
"message": message,
|
|
101
|
+
}
|
|
102
|
+
msg: EventType = make_event(payload, event="Test")
|
|
103
|
+
backend.publish(routing_key, msg)
|
|
104
|
+
click.secho(f"Sent: {msg}")
|
|
105
|
+
backend.disconnect()
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _listen(queues: list[str], payload: bool, pretty: bool, client_name: str) -> None:
|
|
109
|
+
backend = assert_backend()
|
|
110
|
+
|
|
111
|
+
if client_name:
|
|
112
|
+
backend.client_name = client_name
|
|
113
|
+
|
|
114
|
+
def callback(
|
|
115
|
+
queue_name: str, ch: BlockingChannel, method: Basic.Deliver, properties: BasicProperties, body: bytes
|
|
116
|
+
) -> None:
|
|
117
|
+
message: EventType = json.loads(body.decode())
|
|
118
|
+
click.echo(
|
|
119
|
+
f"{Fore.GREEN}{message['timestamp']} "
|
|
120
|
+
f"[{queue_name}]"
|
|
121
|
+
f"{Fore.LIGHTWHITE_EX} [{message['type']}]"
|
|
122
|
+
f"{message['event']} "
|
|
123
|
+
)
|
|
124
|
+
extra: str | JSON
|
|
125
|
+
if payload:
|
|
126
|
+
if pretty:
|
|
127
|
+
extra = json.dumps(message["payload"], indent=4)
|
|
128
|
+
else:
|
|
129
|
+
extra = message["payload"]
|
|
130
|
+
click.echo(f"{Fore.YELLOW}{extra}{Fore.RESET}")
|
|
131
|
+
|
|
132
|
+
try:
|
|
133
|
+
backend.connect()
|
|
134
|
+
_dump_info(backend)
|
|
135
|
+
backend.listen(callback, queues=queues)
|
|
136
|
+
except KeyboardInterrupt:
|
|
137
|
+
click.secho("Stopping listener.", fg="yellow")
|
|
138
|
+
finally:
|
|
139
|
+
backend.disconnect()
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
@cli.command()
|
|
143
|
+
@click.option("-q", "--queues", multiple=True, help="Queue name to listen to")
|
|
144
|
+
@click.option("-c", "--client-name", default=None, help="Override client name")
|
|
145
|
+
@click.option("--payload", default=False, is_flag=True, help="Print payload")
|
|
146
|
+
@click.option("--autoreload", "reload", is_flag=True, help="Enable auto-reloading.")
|
|
147
|
+
@click.option("--pretty", is_flag=True, help="Pretty-print payload.")
|
|
148
|
+
def listen(queues: list[str], payload: bool, reload: bool, pretty: bool, client_name: str) -> None:
|
|
149
|
+
"""Listens for streaming events."""
|
|
150
|
+
if reload:
|
|
151
|
+
from django.utils import autoreload
|
|
152
|
+
|
|
153
|
+
click.secho("Starting listener with autoreload...", fg="yellow")
|
|
154
|
+
autoreload.run_with_reloader(_listen, queues=queues, payload=payload, pretty=pretty, client_name=client_name)
|
|
155
|
+
else:
|
|
156
|
+
_listen(queues=queues, payload=payload, pretty=pretty, client_name=client_name)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
@cli.command()
|
|
160
|
+
def purge() -> None:
|
|
161
|
+
"""Purges all messages from the configured queues."""
|
|
162
|
+
from streaming.backends.rabbitmq import RabbitMQBackend
|
|
163
|
+
from streaming.config import CONFIG
|
|
164
|
+
from streaming.manager import initialize_engine
|
|
165
|
+
|
|
166
|
+
manager = initialize_engine(True)
|
|
167
|
+
backend = manager.backend
|
|
168
|
+
if not isinstance(backend, RabbitMQBackend):
|
|
169
|
+
raise click.ClickException("RabbitMQ backend is not configured. Please set BROKER_URL to a rabbit:// URL.")
|
|
170
|
+
|
|
171
|
+
backend.connect(True)
|
|
172
|
+
for queue_alias, queue_config in CONFIG.QUEUES.items():
|
|
173
|
+
queue_name = queue_config.get("name", queue_alias)
|
|
174
|
+
try:
|
|
175
|
+
message_count = backend.channel.queue_purge(queue_name) # type: ignore[union-attr]
|
|
176
|
+
click.secho(f"Purged {message_count.method.message_count} messages from queue '{queue_name}'.", fg="green")
|
|
177
|
+
except ChannelClosedByBroker:
|
|
178
|
+
click.secho(f"Could not purge queue '{queue_name}'. Queue may not exist.", fg="red")
|
|
179
|
+
backend.disconnect()
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
@cli.command()
|
|
183
|
+
def check() -> None:
|
|
184
|
+
"""Checks the streaming configuration and connection."""
|
|
185
|
+
from streaming.config import CONFIG
|
|
186
|
+
|
|
187
|
+
click.secho("System Configuration:")
|
|
188
|
+
config_dict = dict(CONFIG._parsed)
|
|
189
|
+
for key, value in config_dict.items():
|
|
190
|
+
click.echo(f" {key}: {value}")
|
|
191
|
+
|
|
192
|
+
backend: RabbitMQBackend = assert_backend()
|
|
193
|
+
try:
|
|
194
|
+
backend.connect(True)
|
|
195
|
+
click.secho("Connection successful.", fg="green")
|
|
196
|
+
_dump_info(backend)
|
|
197
|
+
except (StreamingConfigError, AuthorizationError) as e:
|
|
198
|
+
raise ClickException(f"Connection failed: {e}") from e
|
|
@@ -2,13 +2,16 @@ import logging
|
|
|
2
2
|
from typing import TYPE_CHECKING
|
|
3
3
|
from urllib.parse import urlparse
|
|
4
4
|
|
|
5
|
-
from
|
|
5
|
+
from ..exceptions import StreamingConfigError
|
|
6
|
+
from .rabbitmq import RabbitMQBackend
|
|
6
7
|
|
|
7
8
|
if TYPE_CHECKING:
|
|
8
9
|
from ._base import BaseBackend
|
|
9
10
|
|
|
10
11
|
logger = logging.getLogger(__name__)
|
|
11
12
|
|
|
13
|
+
__all__ = ["RabbitMQBackend", "get_backend"]
|
|
14
|
+
|
|
12
15
|
|
|
13
16
|
def get_backend() -> "BaseBackend":
|
|
14
17
|
from streaming.config import CONFIG
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import atexit
|
|
2
|
+
import logging
|
|
3
|
+
from abc import ABC, abstractmethod
|
|
4
|
+
from typing import TYPE_CHECKING, Any
|
|
5
|
+
from urllib.parse import ParseResult, parse_qs, urlparse
|
|
6
|
+
|
|
7
|
+
from streaming.config import CONFIG
|
|
8
|
+
from streaming.utils import get_local_ip
|
|
9
|
+
|
|
10
|
+
from ..exceptions import (
|
|
11
|
+
StreamingError,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from pika.adapters.blocking_connection import BlockingChannel, BlockingConnection
|
|
16
|
+
|
|
17
|
+
from streaming.types import EventType, UserCallback
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
logger = logging.getLogger(__name__)
|
|
21
|
+
|
|
22
|
+
MAX_RETRIES = 5
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class BaseBackend(ABC):
|
|
26
|
+
def __init__(self, url: str) -> None:
|
|
27
|
+
self.connection_url: str = url
|
|
28
|
+
self._parsed_url: ParseResult = urlparse(self.connection_url)
|
|
29
|
+
self._options = {k: v[0] for k, v in parse_qs(self._parsed_url.query).items()}
|
|
30
|
+
self.exchange = self.get_option("exchange", "django-streaming-broadcast")
|
|
31
|
+
|
|
32
|
+
self.host = str(self._parsed_url.hostname)
|
|
33
|
+
self.port = int(self._parsed_url.port) if self._parsed_url.port else 5672
|
|
34
|
+
self._connection: BlockingConnection | None = None
|
|
35
|
+
self._username = self._parsed_url.username or "guest"
|
|
36
|
+
self._password = self._parsed_url.password or "guest"
|
|
37
|
+
|
|
38
|
+
self.channel: BlockingChannel | None = None
|
|
39
|
+
self.exchange = self.get_option("exchange", "django-streaming-broadcast")
|
|
40
|
+
self.retry_exchange = f"retry_{self.exchange}"
|
|
41
|
+
self.timeout = float(self.get_option("timeout", 0.5))
|
|
42
|
+
self.virtual_host = self.get_option("vhost", "/")
|
|
43
|
+
# listener
|
|
44
|
+
self.client_name = CONFIG.CLIENT_NAME or get_local_ip()
|
|
45
|
+
atexit.register(self.disconnect)
|
|
46
|
+
|
|
47
|
+
def get_option(self, name: str, default: Any = "") -> Any:
|
|
48
|
+
return self._options.get(name, default)
|
|
49
|
+
|
|
50
|
+
@property
|
|
51
|
+
def client_name(self) -> str:
|
|
52
|
+
return self.__client_name
|
|
53
|
+
|
|
54
|
+
@client_name.setter
|
|
55
|
+
def client_name(self, name: str) -> None:
|
|
56
|
+
self.__client_name = name
|
|
57
|
+
|
|
58
|
+
def configure_exchanges(self) -> None: # noqa: B027
|
|
59
|
+
pass
|
|
60
|
+
|
|
61
|
+
def configure_queue_routing(self) -> None: # noqa: B027
|
|
62
|
+
pass
|
|
63
|
+
|
|
64
|
+
def get_real_queue_name(self, name: str) -> str:
|
|
65
|
+
return f"{self.client_name}:{name}"
|
|
66
|
+
|
|
67
|
+
def set_credential(self, username: str, password: str) -> None:
|
|
68
|
+
if self._connection:
|
|
69
|
+
raise StreamingError("Disconnect first.")
|
|
70
|
+
self.__username = username
|
|
71
|
+
self.__password = password
|
|
72
|
+
|
|
73
|
+
@abstractmethod
|
|
74
|
+
def connect(self, raise_if_error: bool = False) -> None:
|
|
75
|
+
pass
|
|
76
|
+
|
|
77
|
+
@abstractmethod
|
|
78
|
+
def disconnect(self) -> None:
|
|
79
|
+
pass
|
|
80
|
+
|
|
81
|
+
# Publisher
|
|
82
|
+
@abstractmethod
|
|
83
|
+
def publish(self, routing_key: str, message: "EventType") -> bool: ...
|
|
84
|
+
|
|
85
|
+
@abstractmethod
|
|
86
|
+
def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
|
|
87
|
+
pass
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import sys
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
|
+
|
|
5
|
+
from ._base import BaseBackend
|
|
6
|
+
|
|
7
|
+
logger = logging.getLogger(__name__)
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from ..types import EventType, UserCallback
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class ConsoleBackend(BaseBackend):
|
|
14
|
+
def __init__(self, url: str) -> None:
|
|
15
|
+
super().__init__(url)
|
|
16
|
+
self.stream = self.get_option("stream", "stdout")
|
|
17
|
+
|
|
18
|
+
def publish(self, routing_key: str, message: "EventType") -> bool:
|
|
19
|
+
stream = getattr(sys, self.stream)
|
|
20
|
+
stream.write(f"routing_key:{routing_key} message:{message}\n")
|
|
21
|
+
return True
|
|
22
|
+
|
|
23
|
+
def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
|
|
24
|
+
pass
|
|
25
|
+
|
|
26
|
+
def connect(self, raise_if_error: bool = False) -> None:
|
|
27
|
+
pass
|
|
28
|
+
|
|
29
|
+
def disconnect(self) -> None:
|
|
30
|
+
pass
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import TYPE_CHECKING
|
|
3
|
+
|
|
4
|
+
from ._base import BaseBackend
|
|
5
|
+
|
|
6
|
+
logger = logging.getLogger(__name__)
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from ..types import EventType, UserCallback
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class DebugBackend(BaseBackend):
|
|
13
|
+
def __init__(self, url: str) -> None:
|
|
14
|
+
super().__init__(url)
|
|
15
|
+
self.messages: list[tuple[str, EventType]] = []
|
|
16
|
+
|
|
17
|
+
def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
|
|
18
|
+
pass
|
|
19
|
+
|
|
20
|
+
def connect(self, raise_if_error: bool = False) -> None:
|
|
21
|
+
pass
|
|
22
|
+
|
|
23
|
+
def disconnect(self) -> None:
|
|
24
|
+
pass
|
|
25
|
+
|
|
26
|
+
def publish(self, routing_key: str, message: "EventType") -> bool:
|
|
27
|
+
self.messages.append((routing_key, message))
|
|
28
|
+
return True
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
import socket
|
|
4
|
+
import time
|
|
5
|
+
from typing import TYPE_CHECKING
|
|
6
|
+
|
|
7
|
+
import pika.channel
|
|
8
|
+
import pika.exceptions
|
|
9
|
+
from pika import PlainCredentials
|
|
10
|
+
from pika.exceptions import ConnectionClosedByBroker, ConnectionWrongStateError
|
|
11
|
+
from pika.exchange_type import ExchangeType
|
|
12
|
+
|
|
13
|
+
from streaming.config import CONFIG
|
|
14
|
+
from streaming.utils import exchange_exists
|
|
15
|
+
|
|
16
|
+
from ..exceptions import (
|
|
17
|
+
AuthorizationError,
|
|
18
|
+
StreamingCallbackError,
|
|
19
|
+
StreamingCallbackFailure,
|
|
20
|
+
StreamingConfigError,
|
|
21
|
+
)
|
|
22
|
+
from ..utils import json_dumps
|
|
23
|
+
from ._base import BaseBackend
|
|
24
|
+
|
|
25
|
+
if TYPE_CHECKING:
|
|
26
|
+
from pika.adapters.blocking_connection import BlockingChannel
|
|
27
|
+
from pika.spec import Basic, BasicProperties
|
|
28
|
+
|
|
29
|
+
from streaming.types import EventType, UserCallback
|
|
30
|
+
|
|
31
|
+
logger = logging.getLogger(__name__)
|
|
32
|
+
|
|
33
|
+
MAX_RETRIES = 5
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class Callback:
|
|
37
|
+
def __init__(
|
|
38
|
+
self, queue_name: str, backend: "RabbitMQBackend", user_callback: "UserCallback", ack: bool = True
|
|
39
|
+
) -> None:
|
|
40
|
+
self.backend = backend
|
|
41
|
+
self.user_callback: UserCallback = user_callback
|
|
42
|
+
self.ack = ack
|
|
43
|
+
self.queue_name = queue_name
|
|
44
|
+
|
|
45
|
+
def __call__(
|
|
46
|
+
self, ch: "BlockingChannel", method: "Basic.Deliver", properties: "BasicProperties", body: bytes
|
|
47
|
+
) -> None:
|
|
48
|
+
try:
|
|
49
|
+
self.user_callback(self.queue_name, ch, method, properties, body)
|
|
50
|
+
if self.ack:
|
|
51
|
+
ch.basic_ack(delivery_tag=method.delivery_tag) # type: ignore[arg-type]
|
|
52
|
+
except StreamingCallbackError as e:
|
|
53
|
+
evt: EventType = json.loads(body.decode())
|
|
54
|
+
retries = int(properties.headers.get("x-retries", 0)) # type: ignore[union-attr]
|
|
55
|
+
ch.basic_ack(method.delivery_tag) # type: ignore[arg-type]
|
|
56
|
+
self.backend._handle_retry(evt, ch, method, retries)
|
|
57
|
+
logger.debug("StreamingCallbackError", exc_info=e)
|
|
58
|
+
except StreamingCallbackFailure as e:
|
|
59
|
+
logger.error(f"Callback failure: {e}", exc_info=e)
|
|
60
|
+
except Exception as e:
|
|
61
|
+
logger.exception(f"Unexpected exception occurred: {e}", exc_info=e)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class RabbitMQBackend(BaseBackend):
|
|
65
|
+
def __init__(self, url: str) -> None:
|
|
66
|
+
super().__init__(url)
|
|
67
|
+
|
|
68
|
+
def check_channel(self) -> None:
|
|
69
|
+
if not self.channel:
|
|
70
|
+
raise StreamingConfigError("No active channel")
|
|
71
|
+
|
|
72
|
+
def configure_exchanges(self) -> None:
|
|
73
|
+
logger.debug("Configuring exchanges")
|
|
74
|
+
if not self.channel:
|
|
75
|
+
raise StreamingConfigError("No active channel")
|
|
76
|
+
unrouted_exchange = f"{self.exchange}_unrouted"
|
|
77
|
+
dead_letter_queue = f"{self.exchange}_unrouted_queue"
|
|
78
|
+
logger.debug(f"Declaring exchange '{unrouted_exchange}'")
|
|
79
|
+
self.channel.exchange_declare(exchange=unrouted_exchange, exchange_type=ExchangeType.fanout, durable=True)
|
|
80
|
+
self.channel.queue_declare(queue=dead_letter_queue, durable=True)
|
|
81
|
+
self.channel.queue_bind(exchange=unrouted_exchange, queue=dead_letter_queue)
|
|
82
|
+
|
|
83
|
+
logger.debug(f"Declaring exchange '{self.exchange}'")
|
|
84
|
+
self.channel.exchange_declare(
|
|
85
|
+
exchange=self.exchange,
|
|
86
|
+
exchange_type=ExchangeType.topic,
|
|
87
|
+
durable=True,
|
|
88
|
+
arguments={"alternate-exchange": unrouted_exchange},
|
|
89
|
+
)
|
|
90
|
+
self.channel.exchange_declare(self.retry_exchange, exchange_type=ExchangeType.direct, durable=True)
|
|
91
|
+
|
|
92
|
+
def configure_queue_routing(self) -> None:
|
|
93
|
+
self.check_channel()
|
|
94
|
+
if not exchange_exists(self.channel, self.exchange):
|
|
95
|
+
raise StreamingConfigError("Exchange not found")
|
|
96
|
+
|
|
97
|
+
for alias, config in CONFIG.QUEUES.items():
|
|
98
|
+
real_name = self.get_real_queue_name(alias)
|
|
99
|
+
logger.debug(f"Declaring queue '{real_name}'")
|
|
100
|
+
self.channel.queue_declare(queue=real_name, durable=True, arguments=None) # type: ignore[union-attr]
|
|
101
|
+
binding_keys = config.get("routing", [])
|
|
102
|
+
for binding_key in binding_keys:
|
|
103
|
+
logger.debug("Listening on queue '%s' routed by '%s'", alias, binding_key)
|
|
104
|
+
self.channel.queue_bind(exchange=self.exchange, queue=real_name, routing_key=binding_key) # type: ignore[union-attr]
|
|
105
|
+
|
|
106
|
+
def connect(self, raise_if_error: bool = False) -> None:
|
|
107
|
+
logger.debug("Connecting to %s:%s", self.host, self.port)
|
|
108
|
+
if self._connection and self._connection.is_open:
|
|
109
|
+
self.disconnect()
|
|
110
|
+
|
|
111
|
+
for __ in range(CONFIG.RETRY_COUNT):
|
|
112
|
+
try:
|
|
113
|
+
auth = PlainCredentials(self._username, self._password)
|
|
114
|
+
self._connection = pika.BlockingConnection(
|
|
115
|
+
pika.ConnectionParameters(
|
|
116
|
+
host=self.host,
|
|
117
|
+
port=self.port,
|
|
118
|
+
virtual_host=self.virtual_host,
|
|
119
|
+
credentials=auth,
|
|
120
|
+
socket_timeout=self.timeout,
|
|
121
|
+
blocked_connection_timeout=self.timeout,
|
|
122
|
+
stack_timeout=self.timeout,
|
|
123
|
+
client_properties={
|
|
124
|
+
"connection_name": self.client_name,
|
|
125
|
+
"product": "django-streaming",
|
|
126
|
+
"information": "",
|
|
127
|
+
"version": "1.0",
|
|
128
|
+
},
|
|
129
|
+
)
|
|
130
|
+
)
|
|
131
|
+
self.channel = self._connection.channel()
|
|
132
|
+
return
|
|
133
|
+
except (
|
|
134
|
+
pika.exceptions.AuthenticationError,
|
|
135
|
+
pika.exceptions.ProbableAuthenticationError,
|
|
136
|
+
pika.exceptions.ProbableAccessDeniedError,
|
|
137
|
+
) as e:
|
|
138
|
+
raise AuthorizationError(str(e)) from e
|
|
139
|
+
except (socket.gaierror, pika.exceptions.AMQPError) as e:
|
|
140
|
+
logger.warning(
|
|
141
|
+
f"Could not connect to RabbitMQ. Retrying in {CONFIG.RETRY_DELAY} seconds...",
|
|
142
|
+
)
|
|
143
|
+
time.sleep(CONFIG.RETRY_DELAY)
|
|
144
|
+
if raise_if_error:
|
|
145
|
+
raise StreamingConfigError(f"Error connecting {self.connection_url}") from e
|
|
146
|
+
logger.critical("Could not connect to RabbitMQ after multiple retries.")
|
|
147
|
+
|
|
148
|
+
def disconnect(self) -> None:
|
|
149
|
+
try:
|
|
150
|
+
if self._connection:
|
|
151
|
+
logger.debug("Closing RabbitMQ connection.")
|
|
152
|
+
self._connection.close()
|
|
153
|
+
except (ConnectionClosedByBroker, AttributeError, ConnectionWrongStateError):
|
|
154
|
+
pass
|
|
155
|
+
finally:
|
|
156
|
+
self._connection = None
|
|
157
|
+
self.channel = None
|
|
158
|
+
|
|
159
|
+
# Publisher
|
|
160
|
+
def publish(self, routing_key: str, message: "EventType") -> bool:
|
|
161
|
+
try:
|
|
162
|
+
if not self.channel or self.channel.is_closed:
|
|
163
|
+
self.connect(True)
|
|
164
|
+
logger.debug(f"Publishing to exchange '{self.exchange}' using routing key '{routing_key}'")
|
|
165
|
+
self._basic_publish(message, routing_key, 0)
|
|
166
|
+
return True
|
|
167
|
+
except Exception as e: # noqa: BLE001
|
|
168
|
+
logger.critical("Unhandled error sending to RabbitMQ. Message not published.", exc_info=e)
|
|
169
|
+
return False
|
|
170
|
+
|
|
171
|
+
# Listener
|
|
172
|
+
|
|
173
|
+
def _basic_publish(self, message: "EventType", routing_key: str, retry_count: int = 0) -> None:
|
|
174
|
+
self.check_channel()
|
|
175
|
+
self.channel.basic_publish( # type: ignore[union-attr]
|
|
176
|
+
exchange=self.exchange,
|
|
177
|
+
routing_key=routing_key,
|
|
178
|
+
body=json_dumps(message).encode(),
|
|
179
|
+
properties=pika.BasicProperties(
|
|
180
|
+
delivery_mode=pika.spec.PERSISTENT_DELIVERY_MODE,
|
|
181
|
+
expiration=str(CONFIG.MESSAGE_TTL * 1000), # milliseconds
|
|
182
|
+
headers={"x-retries": retry_count},
|
|
183
|
+
),
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
def _handle_retry(self, message: "EventType", ch: "BlockingChannel", method: "Basic.Deliver", retries: int) -> None:
|
|
187
|
+
ch.basic_ack(method.delivery_tag) # type: ignore[arg-type]
|
|
188
|
+
if retries < MAX_RETRIES:
|
|
189
|
+
delay = 2000 * (2**retries) # ms (exponential backoff)
|
|
190
|
+
delay_queue = f"{self.exchange}_retry_{delay}ms"
|
|
191
|
+
|
|
192
|
+
# Declare a delay queue with TTL
|
|
193
|
+
ch.queue_declare(
|
|
194
|
+
delay_queue,
|
|
195
|
+
durable=True,
|
|
196
|
+
arguments={
|
|
197
|
+
"x-dead-letter-exchange": self.exchange, # after delay → back to main
|
|
198
|
+
"x-message-ttl": delay,
|
|
199
|
+
},
|
|
200
|
+
)
|
|
201
|
+
ch.queue_bind(delay_queue, self.retry_exchange, routing_key="task")
|
|
202
|
+
self._basic_publish(message, routing_key=method.routing_key or "*", retry_count=retries + 1)
|
|
203
|
+
else:
|
|
204
|
+
logger.error(f"Dropping after {MAX_RETRIES} retries")
|
|
205
|
+
|
|
206
|
+
def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
|
|
207
|
+
if self.channel is None:
|
|
208
|
+
self.connect()
|
|
209
|
+
self.check_channel()
|
|
210
|
+
|
|
211
|
+
configured_queues = CONFIG.QUEUES
|
|
212
|
+
if not configured_queues:
|
|
213
|
+
logger.warning("No queues configured in settings.STREAMING['QUEUES']")
|
|
214
|
+
return
|
|
215
|
+
queues_to_listen = queues or configured_queues.keys()
|
|
216
|
+
for queue_name in queues_to_listen:
|
|
217
|
+
if queue_name not in configured_queues:
|
|
218
|
+
logger.error(f"Queue '{queue_name}' not found in configured listening queues. Ignored.")
|
|
219
|
+
continue
|
|
220
|
+
real_queue_name = self.get_real_queue_name(queue_name)
|
|
221
|
+
_callback = Callback(queue_name, self, callback, ack=ack)
|
|
222
|
+
self.channel.basic_consume(queue=real_queue_name, on_message_callback=_callback, auto_ack=False) # type: ignore[union-attr]
|
|
223
|
+
|
|
224
|
+
logger.info("Waiting for messages. To exit press CTRL+C")
|
|
225
|
+
self.channel.start_consuming() # type: ignore[union-attr]
|
|
@@ -8,21 +8,23 @@ from django.dispatch import receiver
|
|
|
8
8
|
|
|
9
9
|
logger = logging.getLogger(__name__)
|
|
10
10
|
|
|
11
|
-
DEFAULT_QUEUE_NAME = "django_model_changes"
|
|
12
|
-
|
|
13
11
|
|
|
14
12
|
class StreamingConfig:
|
|
15
13
|
_DEFAULTS = {
|
|
16
14
|
"BROKER_URL": "console://",
|
|
15
|
+
"QUEUES": {},
|
|
16
|
+
"CLIENT_NAME": "",
|
|
17
17
|
"RETRY_COUNT": 3,
|
|
18
18
|
"RETRY_DELAY": 1,
|
|
19
|
+
"MESSAGE_TTL": 60 * 60 * 24 * 2, # 2 days
|
|
19
20
|
"MANAGER_CLASS": "streaming.manager.ChangeManager",
|
|
20
21
|
}
|
|
21
22
|
|
|
22
23
|
def __init__(self) -> None:
|
|
23
|
-
self._overrides
|
|
24
|
-
self._parsed
|
|
25
|
-
self._cached: dict[str, Any]
|
|
24
|
+
self._overrides: dict[str, Any]
|
|
25
|
+
self._parsed: ChainMap[str, Any]
|
|
26
|
+
self._cached: dict[str, Any]
|
|
27
|
+
self.load()
|
|
26
28
|
|
|
27
29
|
def __getattr__(self, name: str) -> Any:
|
|
28
30
|
if name not in self._parsed:
|
|
@@ -31,12 +33,16 @@ class StreamingConfig:
|
|
|
31
33
|
self._cached[name] = self._parsed[name]
|
|
32
34
|
return self._cached[name]
|
|
33
35
|
|
|
36
|
+
def load(self) -> None:
|
|
37
|
+
self._overrides = getattr(settings, "STREAMING", {})
|
|
38
|
+
self._parsed = ChainMap(self._overrides, self._DEFAULTS)
|
|
39
|
+
self._cached = {}
|
|
40
|
+
|
|
34
41
|
|
|
35
42
|
CONFIG = StreamingConfig()
|
|
36
43
|
|
|
37
44
|
|
|
38
45
|
@receiver(setting_changed)
|
|
39
46
|
def reload_config(sender: Any, setting: str, **kwargs: Any) -> None:
|
|
40
|
-
global CONFIG # noqa: PLW0603
|
|
41
47
|
if setting == "STREAMING":
|
|
42
|
-
CONFIG
|
|
48
|
+
CONFIG.load()
|