django-streaming 0.4__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {django_streaming-0.4 → django_streaming-0.6.0}/PKG-INFO +2 -1
- {django_streaming-0.4 → django_streaming-0.6.0}/pyproject.toml +2 -0
- django_streaming-0.6.0/src/streaming/__cli__.py +202 -0
- {django_streaming-0.4 → django_streaming-0.6.0}/src/streaming/backends/__init__.py +4 -1
- django_streaming-0.6.0/src/streaming/backends/_base.py +90 -0
- django_streaming-0.6.0/src/streaming/backends/console.py +37 -0
- django_streaming-0.6.0/src/streaming/backends/debug.py +35 -0
- django_streaming-0.6.0/src/streaming/backends/rabbitmq.py +232 -0
- {django_streaming-0.4 → django_streaming-0.6.0}/src/streaming/config.py +13 -7
- django_streaming-0.6.0/src/streaming/event.py +64 -0
- {django_streaming-0.4 → django_streaming-0.6.0}/src/streaming/exceptions.py +5 -1
- {django_streaming-0.4 → django_streaming-0.6.0}/src/streaming/manager.py +16 -11
- {django_streaming-0.4 → django_streaming-0.6.0}/src/streaming/types.pyi +5 -7
- django_streaming-0.6.0/src/streaming/utils.py +50 -0
- {django_streaming-0.4 → django_streaming-0.6.0}/src/streaming/version.py +2 -2
- django_streaming-0.4/src/streaming/__cli__.py +0 -158
- django_streaming-0.4/src/streaming/backends/_base.py +0 -26
- django_streaming-0.4/src/streaming/backends/console.py +0 -23
- django_streaming-0.4/src/streaming/backends/debug.py +0 -18
- django_streaming-0.4/src/streaming/backends/rabbitmq.py +0 -200
- django_streaming-0.4/src/streaming/handlers.py +0 -3
- django_streaming-0.4/src/streaming/utils.py +0 -30
- {django_streaming-0.4 → django_streaming-0.6.0}/.gitignore +0 -0
- {django_streaming-0.4 → django_streaming-0.6.0}/LICENSE +0 -0
- {django_streaming-0.4 → django_streaming-0.6.0}/README.md +0 -0
- /django_streaming-0.4/src/streaming/api.py → /django_streaming-0.6.0/src/streaming/__init__.py +0 -0
- {django_streaming-0.4 → django_streaming-0.6.0}/src/streaming/apps.py +0 -0
- {django_streaming-0.4 → django_streaming-0.6.0}/src/streaming/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: django-streaming
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: A project for streaming data.
|
|
5
5
|
Project-URL: Documentation, https://github.com/UNICEF/django-streaming
|
|
6
6
|
Project-URL: Homepage, https://github.com/UNICEF/django-streaming
|
|
@@ -23,6 +23,7 @@ Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
|
|
|
23
23
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
24
24
|
Requires-Python: >=3.10
|
|
25
25
|
Requires-Dist: click>=8
|
|
26
|
+
Requires-Dist: colorama>=0.4.6
|
|
26
27
|
Requires-Dist: django-stubs-ext
|
|
27
28
|
Requires-Dist: django>=3
|
|
28
29
|
Requires-Dist: markdown>=3.8.2
|
|
@@ -35,6 +35,7 @@ dynamic = [
|
|
|
35
35
|
]
|
|
36
36
|
dependencies = [
|
|
37
37
|
"click>=8",
|
|
38
|
+
"colorama>=0.4.6",
|
|
38
39
|
"django>=3",
|
|
39
40
|
"django-stubs-ext",
|
|
40
41
|
"markdown>=3.8.2",
|
|
@@ -52,6 +53,7 @@ dev = [
|
|
|
52
53
|
"django-webtest>=1.9.13",
|
|
53
54
|
"factory-boy>=3.3.3",
|
|
54
55
|
"pre-commit>=3.3.3",
|
|
56
|
+
"pyrabbit2>=1.0.7",
|
|
55
57
|
"pytest>=8.4.1",
|
|
56
58
|
"pytest-cov>=6.2.1",
|
|
57
59
|
"pytest-django>=4.11.1",
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
|
+
|
|
5
|
+
import click
|
|
6
|
+
from click import ClickException
|
|
7
|
+
from colorama import Fore, Style
|
|
8
|
+
from django.core.exceptions import ImproperlyConfigured
|
|
9
|
+
from pika.adapters.blocking_connection import BlockingChannel
|
|
10
|
+
from pika.exceptions import ChannelClosedByBroker
|
|
11
|
+
from pika.spec import Basic, BasicProperties
|
|
12
|
+
|
|
13
|
+
from .backends import RabbitMQBackend, get_backend
|
|
14
|
+
from .event import Event
|
|
15
|
+
from .exceptions import AuthorizationError, StreamingConfigError
|
|
16
|
+
from .utils import make_event
|
|
17
|
+
|
|
18
|
+
if TYPE_CHECKING:
|
|
19
|
+
from .types import JSON
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger(__name__)
|
|
22
|
+
LINE = f"{Fore.YELLOW}%-16s: {Style.RESET_ALL}%s"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _dump_info(backend: "RabbitMQBackend") -> None:
|
|
26
|
+
click.secho(LINE % ("Server", f"{backend.host}:{backend.port}"))
|
|
27
|
+
click.secho(LINE % ("VirtualHost", f"{backend.virtual_host}"))
|
|
28
|
+
click.secho(LINE % ("Exchange", f"{backend.exchange}"))
|
|
29
|
+
click.secho(LINE % ("Timeout", f"{backend.timeout}"))
|
|
30
|
+
click.secho(LINE % ("Client Name", f"{backend.client_name}"))
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def assert_backend() -> "RabbitMQBackend":
|
|
34
|
+
backend: RabbitMQBackend = get_backend() # type: ignore[assignment]
|
|
35
|
+
|
|
36
|
+
if not isinstance(backend, RabbitMQBackend):
|
|
37
|
+
raise ClickException("RabbitMQ backend is not configured")
|
|
38
|
+
return backend
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def configure_logging(debug: bool, loggers: tuple[str, ...] = ("streaming",)) -> None:
|
|
42
|
+
for log_name in loggers:
|
|
43
|
+
logr = logging.getLogger(log_name)
|
|
44
|
+
if debug:
|
|
45
|
+
logr.setLevel(logging.DEBUG)
|
|
46
|
+
logr.addHandler(logging.StreamHandler())
|
|
47
|
+
else:
|
|
48
|
+
logr.handlers = []
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@click.group()
|
|
52
|
+
def cli() -> None:
|
|
53
|
+
"""Streaming CLI."""
|
|
54
|
+
try:
|
|
55
|
+
import django
|
|
56
|
+
|
|
57
|
+
django.setup()
|
|
58
|
+
except ModuleNotFoundError as e:
|
|
59
|
+
raise ClickException(f"Unable to setup Django. {e}") from e
|
|
60
|
+
except ImproperlyConfigured as e:
|
|
61
|
+
raise ClickException("Unable to setup Django. Is DJANGO_SETTINGS_MODULE environment variable set?") from e
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@cli.command()
|
|
65
|
+
@click.option("--queues/--no-queues", "queues", is_flag=True, default=False, help="Debug mode")
|
|
66
|
+
@click.option("--debug", is_flag=True, help="Debug mode")
|
|
67
|
+
def configure(queues: bool = False, debug: bool = False) -> None:
|
|
68
|
+
from streaming.config import CONFIG
|
|
69
|
+
|
|
70
|
+
routing = None
|
|
71
|
+
configure_logging(debug)
|
|
72
|
+
backend: RabbitMQBackend = assert_backend()
|
|
73
|
+
try:
|
|
74
|
+
backend.connect(True)
|
|
75
|
+
backend.configure_exchanges()
|
|
76
|
+
if queues:
|
|
77
|
+
routing = backend.configure_queue_routing()
|
|
78
|
+
check.callback() # type: ignore[misc]
|
|
79
|
+
if routing:
|
|
80
|
+
click.secho(LINE % ("Queues", ""))
|
|
81
|
+
for k, v in routing.items():
|
|
82
|
+
click.secho(LINE % (f" {k}", "; ".join(v)))
|
|
83
|
+
except AuthorizationError as e:
|
|
84
|
+
click.secho(f"Unable to connect using {CONFIG.BROKER_URL}", fg="red", err=True)
|
|
85
|
+
raise ClickException(str(e)) from e
|
|
86
|
+
except StreamingConfigError as e:
|
|
87
|
+
click.secho(f"Generic error {e}", fg="red", err=True)
|
|
88
|
+
raise ClickException(str(e)) from e
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@cli.command()
|
|
92
|
+
@click.argument("routing_key")
|
|
93
|
+
@click.option("-c", "--client-name", default=None, help="Override client name")
|
|
94
|
+
@click.option("--message", default="Test Message", help="Message to send")
|
|
95
|
+
@click.option("--debug", is_flag=True, help="Debug mode")
|
|
96
|
+
def send(routing_key: str, message: str, client_name: str, debug: bool) -> None:
|
|
97
|
+
backend = assert_backend()
|
|
98
|
+
configure_logging(debug)
|
|
99
|
+
|
|
100
|
+
if client_name:
|
|
101
|
+
backend.client_name = client_name
|
|
102
|
+
try:
|
|
103
|
+
payload = json.loads(message)
|
|
104
|
+
except json.decoder.JSONDecodeError:
|
|
105
|
+
payload = {
|
|
106
|
+
"message": message,
|
|
107
|
+
}
|
|
108
|
+
msg: Event = make_event(payload, key="Test")
|
|
109
|
+
backend.publish(routing_key, msg)
|
|
110
|
+
click.secho(f"Sent: {msg}")
|
|
111
|
+
backend.disconnect()
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _listen(queues: list[str], payload: bool, pretty: bool) -> None:
|
|
115
|
+
backend = assert_backend()
|
|
116
|
+
|
|
117
|
+
def callback(
|
|
118
|
+
queue_name: str, ch: BlockingChannel, method: Basic.Deliver, properties: BasicProperties, body: bytes
|
|
119
|
+
) -> None:
|
|
120
|
+
message: Event = Event.unmarshal(body)
|
|
121
|
+
click.echo(f"{Fore.GREEN}{message.timestamp} [{queue_name}]{Fore.LIGHTWHITE_EX} [{message.key}] {message.id} ")
|
|
122
|
+
extra: str | JSON
|
|
123
|
+
if payload:
|
|
124
|
+
if pretty:
|
|
125
|
+
extra = json.dumps(message.payload, indent=4)
|
|
126
|
+
else:
|
|
127
|
+
extra = message.payload
|
|
128
|
+
click.echo(f"{Fore.YELLOW}{extra}{Fore.RESET}")
|
|
129
|
+
|
|
130
|
+
try:
|
|
131
|
+
backend.connect(True)
|
|
132
|
+
_dump_info(backend)
|
|
133
|
+
backend.listen(callback, queues=queues)
|
|
134
|
+
except ChannelClosedByBroker as e:
|
|
135
|
+
click.secho(str(e), fg="red")
|
|
136
|
+
if "no queue" in str(e):
|
|
137
|
+
click.secho("Did you run 'stream configure --queues'", fg="red")
|
|
138
|
+
except StreamingConfigError as e:
|
|
139
|
+
click.secho(str(e), fg="red")
|
|
140
|
+
click.get_current_context().exit(2)
|
|
141
|
+
except KeyboardInterrupt:
|
|
142
|
+
click.secho("Stopping listener.", fg="yellow")
|
|
143
|
+
finally:
|
|
144
|
+
backend.disconnect()
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
@cli.command()
|
|
148
|
+
@click.option("-q", "--queues", multiple=True, help="Queue name to listen to")
|
|
149
|
+
@click.option("--payload", default=False, is_flag=True, help="Print payload")
|
|
150
|
+
@click.option("--autoreload", "reload", is_flag=True, help="Enable auto-reloading.")
|
|
151
|
+
@click.option("--pretty", is_flag=True, help="Pretty-print payload.")
|
|
152
|
+
def listen(queues: list[str], payload: bool, reload: bool, pretty: bool) -> None:
|
|
153
|
+
"""Listens for streaming events."""
|
|
154
|
+
if reload:
|
|
155
|
+
from django.utils import autoreload
|
|
156
|
+
|
|
157
|
+
click.secho("Starting listener with autoreload...", fg="yellow")
|
|
158
|
+
autoreload.run_with_reloader(_listen, queues=queues, payload=payload, pretty=pretty)
|
|
159
|
+
else:
|
|
160
|
+
_listen(queues=queues, payload=payload, pretty=pretty)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
@cli.command()
|
|
164
|
+
def purge() -> None:
|
|
165
|
+
"""Purges all messages from the configured queues."""
|
|
166
|
+
from streaming.backends.rabbitmq import RabbitMQBackend
|
|
167
|
+
from streaming.config import CONFIG
|
|
168
|
+
from streaming.manager import initialize_engine
|
|
169
|
+
|
|
170
|
+
manager = initialize_engine(True)
|
|
171
|
+
backend = manager.backend
|
|
172
|
+
if not isinstance(backend, RabbitMQBackend):
|
|
173
|
+
raise click.ClickException("RabbitMQ backend is not configured. Please set BROKER_URL to a rabbit:// URL.")
|
|
174
|
+
|
|
175
|
+
backend.connect(True)
|
|
176
|
+
for queue_alias, queue_config in CONFIG.QUEUES.items():
|
|
177
|
+
queue_name = queue_config.get("name", queue_alias)
|
|
178
|
+
try:
|
|
179
|
+
message_count = backend.channel.queue_purge(queue_name) # type: ignore[union-attr]
|
|
180
|
+
click.secho(f"Purged {message_count.method.message_count} messages from queue '{queue_name}'.", fg="green")
|
|
181
|
+
except ChannelClosedByBroker:
|
|
182
|
+
click.secho(f"Could not purge queue '{queue_name}'. Queue may not exist.", fg="red")
|
|
183
|
+
backend.disconnect()
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
@cli.command()
|
|
187
|
+
def check() -> None:
|
|
188
|
+
"""Checks the streaming configuration and connection."""
|
|
189
|
+
from streaming.config import CONFIG
|
|
190
|
+
|
|
191
|
+
click.secho("System Configuration:")
|
|
192
|
+
config_dict = dict(CONFIG._parsed)
|
|
193
|
+
for key, value in config_dict.items():
|
|
194
|
+
click.echo(f" {key}: {value}")
|
|
195
|
+
|
|
196
|
+
backend: RabbitMQBackend = assert_backend()
|
|
197
|
+
try:
|
|
198
|
+
backend.connect(True)
|
|
199
|
+
click.secho("Connection successful.", fg="green")
|
|
200
|
+
_dump_info(backend)
|
|
201
|
+
except (StreamingConfigError, AuthorizationError) as e:
|
|
202
|
+
raise ClickException(f"Connection failed: {e}") from e
|
|
@@ -2,13 +2,16 @@ import logging
|
|
|
2
2
|
from typing import TYPE_CHECKING
|
|
3
3
|
from urllib.parse import urlparse
|
|
4
4
|
|
|
5
|
-
from
|
|
5
|
+
from ..exceptions import StreamingConfigError
|
|
6
|
+
from .rabbitmq import RabbitMQBackend
|
|
6
7
|
|
|
7
8
|
if TYPE_CHECKING:
|
|
8
9
|
from ._base import BaseBackend
|
|
9
10
|
|
|
10
11
|
logger = logging.getLogger(__name__)
|
|
11
12
|
|
|
13
|
+
__all__ = ["RabbitMQBackend", "get_backend"]
|
|
14
|
+
|
|
12
15
|
|
|
13
16
|
def get_backend() -> "BaseBackend":
|
|
14
17
|
from streaming.config import CONFIG
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import atexit
|
|
2
|
+
import logging
|
|
3
|
+
from abc import ABC, abstractmethod
|
|
4
|
+
from typing import TYPE_CHECKING, Any
|
|
5
|
+
from urllib.parse import ParseResult, parse_qs, urlparse
|
|
6
|
+
|
|
7
|
+
from streaming.config import CONFIG
|
|
8
|
+
from streaming.utils import get_local_ip
|
|
9
|
+
|
|
10
|
+
from ..exceptions import (
|
|
11
|
+
StreamingError,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from pika.adapters.blocking_connection import BlockingChannel, BlockingConnection
|
|
16
|
+
|
|
17
|
+
from ..event import Event
|
|
18
|
+
from ..types import UserCallback
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger(__name__)
|
|
22
|
+
|
|
23
|
+
MAX_RETRIES = 5
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class BaseBackend(ABC):
|
|
27
|
+
def __init__(self, url: str) -> None:
|
|
28
|
+
self.connection_url: str = url
|
|
29
|
+
self._parsed_url: ParseResult = urlparse(self.connection_url)
|
|
30
|
+
self._options = {k: v[0] for k, v in parse_qs(self._parsed_url.query).items()}
|
|
31
|
+
self.exchange = self.get_option("exchange", "django-streaming-broadcast")
|
|
32
|
+
|
|
33
|
+
self.host = str(self._parsed_url.hostname)
|
|
34
|
+
self.port = int(self._parsed_url.port) if self._parsed_url.port else 5672
|
|
35
|
+
self._connection: BlockingConnection | None = None
|
|
36
|
+
self._username = self._parsed_url.username or "guest"
|
|
37
|
+
self._password = self._parsed_url.password or "guest"
|
|
38
|
+
|
|
39
|
+
self.channel: BlockingChannel | None = None
|
|
40
|
+
self.exchange = self.get_option("exchange", "django-streaming-broadcast")
|
|
41
|
+
self.retry_exchange = f"retry_{self.exchange}"
|
|
42
|
+
self.timeout = float(self.get_option("timeout", 0.5))
|
|
43
|
+
self.virtual_host = self.get_option("vhost", "/")
|
|
44
|
+
# listener
|
|
45
|
+
self.client_name = CONFIG.CLIENT_NAME or self.get_option("client_name", get_local_ip())
|
|
46
|
+
atexit.register(self.disconnect)
|
|
47
|
+
|
|
48
|
+
def get_option(self, name: str, default: Any = "") -> Any:
|
|
49
|
+
return self._options.get(name, default)
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
def client_name(self) -> str:
|
|
53
|
+
return self.__client_name
|
|
54
|
+
|
|
55
|
+
@client_name.setter
|
|
56
|
+
def client_name(self, name: str) -> None:
|
|
57
|
+
self.__client_name = name
|
|
58
|
+
|
|
59
|
+
@abstractmethod
|
|
60
|
+
def configure_exchanges(self) -> None: # noqa: B027
|
|
61
|
+
...
|
|
62
|
+
|
|
63
|
+
@abstractmethod
|
|
64
|
+
def configure_queue_routing(self) -> dict[str, list[str]]: # noqa: B027
|
|
65
|
+
...
|
|
66
|
+
|
|
67
|
+
def get_real_queue_name(self, name: str) -> str:
|
|
68
|
+
return f"{self.client_name}:{name}"
|
|
69
|
+
|
|
70
|
+
def set_credential(self, username: str, password: str) -> None:
|
|
71
|
+
if self._connection:
|
|
72
|
+
raise StreamingError("Disconnect first.")
|
|
73
|
+
self.__username = username
|
|
74
|
+
self.__password = password
|
|
75
|
+
|
|
76
|
+
@abstractmethod
|
|
77
|
+
def connect(self, raise_if_error: bool = False) -> None:
|
|
78
|
+
pass
|
|
79
|
+
|
|
80
|
+
@abstractmethod
|
|
81
|
+
def disconnect(self) -> None:
|
|
82
|
+
pass
|
|
83
|
+
|
|
84
|
+
# Publisher
|
|
85
|
+
@abstractmethod
|
|
86
|
+
def publish(self, routing_key: str, message: "Event") -> bool: ...
|
|
87
|
+
|
|
88
|
+
@abstractmethod
|
|
89
|
+
def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
|
|
90
|
+
pass
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import sys
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
|
+
|
|
5
|
+
from ._base import BaseBackend
|
|
6
|
+
|
|
7
|
+
logger = logging.getLogger(__name__)
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from ..event import Event
|
|
11
|
+
from ..types import UserCallback
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ConsoleBackend(BaseBackend):
|
|
15
|
+
def __init__(self, url: str) -> None:
|
|
16
|
+
super().__init__(url)
|
|
17
|
+
self.stream = self.get_option("stream", "stdout")
|
|
18
|
+
|
|
19
|
+
def configure_queue_routing(self) -> dict[str, list[str]]: # pragma: no cover
|
|
20
|
+
return {}
|
|
21
|
+
|
|
22
|
+
def configure_exchanges(self) -> None: # pragma: no cover
|
|
23
|
+
pass
|
|
24
|
+
|
|
25
|
+
def publish(self, routing_key: str, message: "Event") -> bool:
|
|
26
|
+
stream = getattr(sys, self.stream)
|
|
27
|
+
stream.write(f"routing_key:{routing_key} message:{message}\n")
|
|
28
|
+
return True
|
|
29
|
+
|
|
30
|
+
def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
|
|
31
|
+
pass
|
|
32
|
+
|
|
33
|
+
def connect(self, raise_if_error: bool = False) -> None:
|
|
34
|
+
pass
|
|
35
|
+
|
|
36
|
+
def disconnect(self) -> None:
|
|
37
|
+
pass
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import TYPE_CHECKING
|
|
3
|
+
|
|
4
|
+
from ._base import BaseBackend
|
|
5
|
+
|
|
6
|
+
logger = logging.getLogger(__name__)
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from ..event import Event
|
|
10
|
+
from ..types import UserCallback
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class DebugBackend(BaseBackend):
|
|
14
|
+
def configure_queue_routing(self) -> dict[str, list[str]]: # pragma: no cover
|
|
15
|
+
return {}
|
|
16
|
+
|
|
17
|
+
def configure_exchanges(self) -> None: # pragma: no cover
|
|
18
|
+
pass
|
|
19
|
+
|
|
20
|
+
def __init__(self, url: str) -> None:
|
|
21
|
+
super().__init__(url)
|
|
22
|
+
self.messages: list[tuple[str, Event]] = []
|
|
23
|
+
|
|
24
|
+
def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
def connect(self, raise_if_error: bool = False) -> None:
|
|
28
|
+
pass
|
|
29
|
+
|
|
30
|
+
def disconnect(self) -> None:
|
|
31
|
+
pass
|
|
32
|
+
|
|
33
|
+
def publish(self, routing_key: str, message: "Event") -> bool:
|
|
34
|
+
self.messages.append((routing_key, message))
|
|
35
|
+
return True
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import socket
|
|
3
|
+
import time
|
|
4
|
+
from collections import defaultdict
|
|
5
|
+
from contextlib import suppress
|
|
6
|
+
from typing import TYPE_CHECKING
|
|
7
|
+
|
|
8
|
+
import pika.channel
|
|
9
|
+
import pika.exceptions
|
|
10
|
+
from pika import PlainCredentials
|
|
11
|
+
from pika.exceptions import ConnectionClosedByBroker, ConnectionWrongStateError
|
|
12
|
+
from pika.exchange_type import ExchangeType
|
|
13
|
+
|
|
14
|
+
from streaming.config import CONFIG
|
|
15
|
+
from streaming.utils import exchange_exists
|
|
16
|
+
|
|
17
|
+
from ..event import Event
|
|
18
|
+
from ..exceptions import (
|
|
19
|
+
AuthorizationError,
|
|
20
|
+
StreamingCallbackFailure,
|
|
21
|
+
StreamingCallbackRetryError,
|
|
22
|
+
StreamingConfigError,
|
|
23
|
+
)
|
|
24
|
+
from ._base import BaseBackend
|
|
25
|
+
|
|
26
|
+
if TYPE_CHECKING:
|
|
27
|
+
from pika.adapters.blocking_connection import BlockingChannel
|
|
28
|
+
from pika.spec import Basic, BasicProperties
|
|
29
|
+
|
|
30
|
+
from streaming.types import UserCallback
|
|
31
|
+
|
|
32
|
+
logger = logging.getLogger(__name__)
|
|
33
|
+
|
|
34
|
+
MAX_RETRIES = 5
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class Callback:
|
|
38
|
+
def __init__(
|
|
39
|
+
self, queue_name: str, backend: "RabbitMQBackend", user_callback: "UserCallback", ack: bool = True
|
|
40
|
+
) -> None:
|
|
41
|
+
self.backend = backend
|
|
42
|
+
self.user_callback: UserCallback = user_callback
|
|
43
|
+
self.ack = ack
|
|
44
|
+
self.queue_name = queue_name
|
|
45
|
+
|
|
46
|
+
def __call__(
|
|
47
|
+
self, ch: "BlockingChannel", method: "Basic.Deliver", properties: "BasicProperties", body: bytes
|
|
48
|
+
) -> None:
|
|
49
|
+
try:
|
|
50
|
+
self.user_callback(self.queue_name, ch, method, properties, body)
|
|
51
|
+
if self.ack:
|
|
52
|
+
ch.basic_ack(delivery_tag=method.delivery_tag) # type: ignore[arg-type]
|
|
53
|
+
except StreamingCallbackRetryError as e:
|
|
54
|
+
evt: Event = Event.unmarshal(body)
|
|
55
|
+
logger.debug(f"StreamingCallbackError {evt.id}", exc_info=e)
|
|
56
|
+
retries = int(properties.headers.get("x-retries", 0)) # type: ignore[union-attr]
|
|
57
|
+
self.backend._handle_retry(evt, ch, method, retries)
|
|
58
|
+
except StreamingCallbackFailure as e:
|
|
59
|
+
logger.error(f"Callback failure: {e}", exc_info=e)
|
|
60
|
+
except Exception as e:
|
|
61
|
+
logger.exception(f"Unexpected exception occurred: {e}", exc_info=e)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class RabbitMQBackend(BaseBackend):
|
|
65
|
+
def __init__(self, url: str) -> None:
|
|
66
|
+
super().__init__(url)
|
|
67
|
+
|
|
68
|
+
def check_channel(self) -> None:
|
|
69
|
+
if not self.channel:
|
|
70
|
+
raise StreamingConfigError("No active channel")
|
|
71
|
+
|
|
72
|
+
def configure_exchanges(self) -> None:
|
|
73
|
+
logger.debug("Configuring exchanges")
|
|
74
|
+
if not self.channel:
|
|
75
|
+
raise StreamingConfigError("No active channel")
|
|
76
|
+
unrouted_exchange = f"{self.exchange}_unrouted"
|
|
77
|
+
dead_letter_queue = f"{self.exchange}_unrouted_queue"
|
|
78
|
+
logger.debug(f"Declaring exchange '{unrouted_exchange}'")
|
|
79
|
+
self.channel.exchange_declare(exchange=unrouted_exchange, exchange_type=ExchangeType.fanout, durable=True)
|
|
80
|
+
self.channel.queue_declare(queue=dead_letter_queue, durable=True)
|
|
81
|
+
self.channel.queue_bind(exchange=unrouted_exchange, queue=dead_letter_queue)
|
|
82
|
+
|
|
83
|
+
logger.debug(f"Declaring exchange '{self.exchange}'")
|
|
84
|
+
self.channel.exchange_declare(
|
|
85
|
+
exchange=self.exchange,
|
|
86
|
+
exchange_type=ExchangeType.topic,
|
|
87
|
+
durable=True,
|
|
88
|
+
arguments={"alternate-exchange": unrouted_exchange},
|
|
89
|
+
)
|
|
90
|
+
self.channel.exchange_declare(self.retry_exchange, exchange_type=ExchangeType.direct, durable=True)
|
|
91
|
+
|
|
92
|
+
def configure_queue_routing(self) -> dict[str, list[str]]:
|
|
93
|
+
self.check_channel()
|
|
94
|
+
if not exchange_exists(self.channel, self.exchange):
|
|
95
|
+
raise StreamingConfigError("Exchange not found")
|
|
96
|
+
ret = defaultdict(list)
|
|
97
|
+
for alias, config in CONFIG.QUEUES.items():
|
|
98
|
+
real_name = self.get_real_queue_name(alias)
|
|
99
|
+
logger.debug(f"Declaring queue '{real_name}'")
|
|
100
|
+
self.channel.queue_declare(queue=real_name, durable=True, arguments=None) # type: ignore[union-attr]
|
|
101
|
+
if binding_keys := config.get("routing", []):
|
|
102
|
+
for binding_key in binding_keys:
|
|
103
|
+
logger.debug("Listening on queue '%s' routed by '%s'", alias, binding_key)
|
|
104
|
+
self.channel.queue_bind(exchange=self.exchange, queue=real_name, routing_key=binding_key) # type: ignore[union-attr]
|
|
105
|
+
ret[alias].append(binding_key)
|
|
106
|
+
else:
|
|
107
|
+
ret[alias].append("warning no routing defined")
|
|
108
|
+
return ret
|
|
109
|
+
|
|
110
|
+
def connect(self, raise_if_error: bool = False) -> None:
|
|
111
|
+
logger.debug("Connecting to %s:%s", self.host, self.port)
|
|
112
|
+
if self._connection and self._connection.is_open:
|
|
113
|
+
self.disconnect()
|
|
114
|
+
|
|
115
|
+
for __ in range(CONFIG.RETRY_COUNT):
|
|
116
|
+
try:
|
|
117
|
+
auth = PlainCredentials(self._username, self._password)
|
|
118
|
+
self._connection = pika.BlockingConnection(
|
|
119
|
+
pika.ConnectionParameters(
|
|
120
|
+
host=self.host,
|
|
121
|
+
port=self.port,
|
|
122
|
+
virtual_host=self.virtual_host,
|
|
123
|
+
credentials=auth,
|
|
124
|
+
socket_timeout=self.timeout,
|
|
125
|
+
blocked_connection_timeout=self.timeout,
|
|
126
|
+
stack_timeout=self.timeout,
|
|
127
|
+
client_properties={
|
|
128
|
+
"connection_name": self.client_name,
|
|
129
|
+
"product": "django-streaming",
|
|
130
|
+
"information": "",
|
|
131
|
+
"version": "1.0",
|
|
132
|
+
},
|
|
133
|
+
)
|
|
134
|
+
)
|
|
135
|
+
self.channel = self._connection.channel()
|
|
136
|
+
return
|
|
137
|
+
except (
|
|
138
|
+
pika.exceptions.AuthenticationError,
|
|
139
|
+
pika.exceptions.ProbableAuthenticationError,
|
|
140
|
+
pika.exceptions.ProbableAccessDeniedError,
|
|
141
|
+
) as e:
|
|
142
|
+
raise AuthorizationError(str(e)) from e
|
|
143
|
+
except (socket.gaierror, pika.exceptions.AMQPError) as e:
|
|
144
|
+
logger.warning(
|
|
145
|
+
f"Could not connect to RabbitMQ. Retrying in {CONFIG.RETRY_DELAY} seconds...",
|
|
146
|
+
)
|
|
147
|
+
time.sleep(CONFIG.RETRY_DELAY)
|
|
148
|
+
if raise_if_error:
|
|
149
|
+
raise StreamingConfigError(f"Error connecting {self.connection_url}") from e
|
|
150
|
+
logger.critical("Could not connect to RabbitMQ after multiple retries.")
|
|
151
|
+
|
|
152
|
+
def disconnect(self) -> None:
|
|
153
|
+
try:
|
|
154
|
+
if self._connection:
|
|
155
|
+
with suppress(ConnectionClosedByBroker, AttributeError, ConnectionWrongStateError):
|
|
156
|
+
logger.debug("Closing RabbitMQ connection.")
|
|
157
|
+
self._connection.close()
|
|
158
|
+
finally:
|
|
159
|
+
self._connection = None
|
|
160
|
+
self.channel = None
|
|
161
|
+
|
|
162
|
+
# Publisher
|
|
163
|
+
def publish(self, routing_key: str, message: "Event") -> bool:
|
|
164
|
+
try:
|
|
165
|
+
if not self.channel or self.channel.is_closed:
|
|
166
|
+
self.connect(True)
|
|
167
|
+
logger.debug(f"Publishing to exchange '{self.exchange}' using routing key '{routing_key}'")
|
|
168
|
+
self._basic_publish(message, routing_key, 0)
|
|
169
|
+
return True
|
|
170
|
+
except Exception as e: # noqa: BLE001
|
|
171
|
+
logger.critical("Unhandled error sending to RabbitMQ. Message not published.", exc_info=e)
|
|
172
|
+
return False
|
|
173
|
+
|
|
174
|
+
# Listener
|
|
175
|
+
|
|
176
|
+
def _basic_publish(
|
|
177
|
+
self, message: "Event", routing_key: str, retry_count: int = 0, exchange: str | None = None
|
|
178
|
+
) -> None:
|
|
179
|
+
self.check_channel()
|
|
180
|
+
self.channel.basic_publish( # type: ignore[union-attr]
|
|
181
|
+
exchange=exchange or self.exchange,
|
|
182
|
+
routing_key=routing_key,
|
|
183
|
+
body=message.marshall(),
|
|
184
|
+
properties=pika.BasicProperties(
|
|
185
|
+
delivery_mode=pika.spec.PERSISTENT_DELIVERY_MODE,
|
|
186
|
+
expiration=str(CONFIG.MESSAGE_TTL * 1000), # milliseconds
|
|
187
|
+
headers={"x-retries": retry_count},
|
|
188
|
+
),
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
def _handle_retry(self, message: "Event", ch: "BlockingChannel", method: "Basic.Deliver", retries: int) -> None:
|
|
192
|
+
ch.basic_ack(method.delivery_tag) # type: ignore[arg-type]
|
|
193
|
+
if retries < MAX_RETRIES:
|
|
194
|
+
delay = 2000 * (2**retries) # ms (exponential backoff)
|
|
195
|
+
delay_queue = f"{self.exchange}_retry_{method.routing_key}_{delay}ms"
|
|
196
|
+
logger.debug(
|
|
197
|
+
f"Retrying message {message.id} ({retries + 1}/{MAX_RETRIES}) "
|
|
198
|
+
f"with routing key '{method.routing_key}' in {delay / 1000}s"
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
# Declare a delay queue with TTL
|
|
202
|
+
ch.queue_declare(
|
|
203
|
+
delay_queue,
|
|
204
|
+
durable=True,
|
|
205
|
+
arguments={
|
|
206
|
+
"x-dead-letter-exchange": self.exchange, # after delay → back to main
|
|
207
|
+
"x-message-ttl": delay,
|
|
208
|
+
"x-dead-letter-routing-key": method.routing_key,
|
|
209
|
+
},
|
|
210
|
+
)
|
|
211
|
+
ch.queue_bind(delay_queue, self.retry_exchange, routing_key=delay_queue)
|
|
212
|
+
self._basic_publish(message, routing_key=delay_queue, retry_count=retries + 1, exchange=self.retry_exchange)
|
|
213
|
+
else:
|
|
214
|
+
logger.error(f"Dropping message {message.id} after {MAX_RETRIES} retries")
|
|
215
|
+
|
|
216
|
+
def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
|
|
217
|
+
if self.channel is None:
|
|
218
|
+
self.connect()
|
|
219
|
+
self.check_channel()
|
|
220
|
+
|
|
221
|
+
configured_queues = CONFIG.QUEUES
|
|
222
|
+
if not configured_queues:
|
|
223
|
+
logger.warning("No queues configured in settings.STREAMING['QUEUES']")
|
|
224
|
+
return
|
|
225
|
+
queues_to_listen = queues or configured_queues.keys()
|
|
226
|
+
for queue_name in queues_to_listen:
|
|
227
|
+
real_queue_name = self.get_real_queue_name(queue_name)
|
|
228
|
+
_callback = Callback(queue_name, self, callback, ack=ack)
|
|
229
|
+
self.channel.basic_consume(queue=real_queue_name, on_message_callback=_callback, auto_ack=False) # type: ignore[union-attr]
|
|
230
|
+
|
|
231
|
+
logger.info("Waiting for messages. To exit press CTRL+C")
|
|
232
|
+
self.channel.start_consuming() # type: ignore[union-attr]
|