django-streaming 0.3__tar.gz → 0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {django_streaming-0.3 → django_streaming-0.4}/PKG-INFO +6 -9
- {django_streaming-0.3 → django_streaming-0.4}/README.md +1 -1
- {django_streaming-0.3 → django_streaming-0.4}/pyproject.toml +4 -10
- {django_streaming-0.3 → django_streaming-0.4}/src/streaming/__cli__.py +6 -7
- {django_streaming-0.3 → django_streaming-0.4}/src/streaming/apps.py +0 -5
- {django_streaming-0.3 → django_streaming-0.4}/src/streaming/backends/__init__.py +5 -5
- {django_streaming-0.3 → django_streaming-0.4}/src/streaming/backends/_base.py +2 -2
- {django_streaming-0.3 → django_streaming-0.4}/src/streaming/backends/console.py +5 -2
- {django_streaming-0.3 → django_streaming-0.4}/src/streaming/backends/debug.py +2 -2
- django_streaming-0.4/src/streaming/backends/rabbitmq.py +200 -0
- {django_streaming-0.3 → django_streaming-0.4}/src/streaming/config.py +1 -1
- django_streaming-0.4/src/streaming/exceptions.py +23 -0
- {django_streaming-0.3 → django_streaming-0.4}/src/streaming/utils.py +8 -0
- {django_streaming-0.3 → django_streaming-0.4}/src/streaming/version.py +2 -2
- django_streaming-0.3/src/streaming/backends/rabbitmq.py +0 -127
- django_streaming-0.3/src/streaming/backends/redis.py +0 -56
- django_streaming-0.3/src/streaming/exceptions.py +0 -15
- {django_streaming-0.3 → django_streaming-0.4}/.gitignore +0 -0
- {django_streaming-0.3 → django_streaming-0.4}/LICENSE +0 -0
- {django_streaming-0.3 → django_streaming-0.4}/src/streaming/api.py +0 -0
- {django_streaming-0.3 → django_streaming-0.4}/src/streaming/handlers.py +0 -0
- {django_streaming-0.3 → django_streaming-0.4}/src/streaming/manager.py +0 -0
- {django_streaming-0.3 → django_streaming-0.4}/src/streaming/py.typed +0 -0
- {django_streaming-0.3 → django_streaming-0.4}/src/streaming/types.pyi +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: django-streaming
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4
|
|
4
4
|
Summary: A project for streaming data.
|
|
5
5
|
Project-URL: Documentation, https://github.com/UNICEF/django-streaming
|
|
6
6
|
Project-URL: Homepage, https://github.com/UNICEF/django-streaming
|
|
@@ -8,29 +8,26 @@ Author-email: Stefano Apostolico <s.apostolico@gmail.com>
|
|
|
8
8
|
License-File: LICENSE
|
|
9
9
|
Classifier: Environment :: Web Environment
|
|
10
10
|
Classifier: Framework :: Django
|
|
11
|
+
Classifier: Framework :: Django :: 3.2
|
|
11
12
|
Classifier: Framework :: Django :: 4.2
|
|
12
13
|
Classifier: Framework :: Django :: 5.2
|
|
13
14
|
Classifier: Intended Audience :: Developers
|
|
14
15
|
Classifier: License :: OSI Approved :: MIT License
|
|
15
16
|
Classifier: Operating System :: OS Independent
|
|
16
17
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
18
18
|
Classifier: Programming Language :: Python :: 3.10
|
|
19
19
|
Classifier: Programming Language :: Python :: 3.11
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.12
|
|
21
21
|
Classifier: Programming Language :: Python :: 3.13
|
|
22
22
|
Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
|
|
23
23
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
24
|
-
Requires-Python: >=3.
|
|
24
|
+
Requires-Python: >=3.10
|
|
25
25
|
Requires-Dist: click>=8
|
|
26
26
|
Requires-Dist: django-stubs-ext
|
|
27
|
-
Requires-Dist: django>=
|
|
27
|
+
Requires-Dist: django>=3
|
|
28
28
|
Requires-Dist: markdown>=3.8.2
|
|
29
|
+
Requires-Dist: pika>=1.3.2
|
|
29
30
|
Requires-Dist: requests>=2.32.5
|
|
30
|
-
Provides-Extra: rabbitmq
|
|
31
|
-
Requires-Dist: pika>=1.3.2; extra == 'rabbitmq'
|
|
32
|
-
Provides-Extra: redis
|
|
33
|
-
Requires-Dist: redis>=5; extra == 'redis'
|
|
34
31
|
Description-Content-Type: text/markdown
|
|
35
32
|
|
|
36
33
|
# Django Streaming
|
|
@@ -45,7 +42,7 @@ Description-Content-Type: text/markdown
|
|
|
45
42
|
`django-streaming` is a robust and flexible Django application designed for efficiently streaming data from your
|
|
46
43
|
Django projects to various message brokers.
|
|
47
44
|
|
|
48
|
-
It provides a pluggable backend architecture supporting popular options like RabbitMQ
|
|
45
|
+
It provides a pluggable backend architecture supporting popular options like RabbitMQ, alongside console and
|
|
49
46
|
debug backends for development. With features like a threaded change manager for non-blocking asynchronous
|
|
50
47
|
notifications, configurable connection retry mechanisms, and graceful shutdown handling, django-streaming ensures
|
|
51
48
|
reliable data delivery. It seamlessly integrates with Django's ORM via signals to automatically stream model changes
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
`django-streaming` is a robust and flexible Django application designed for efficiently streaming data from your
|
|
11
11
|
Django projects to various message brokers.
|
|
12
12
|
|
|
13
|
-
It provides a pluggable backend architecture supporting popular options like RabbitMQ
|
|
13
|
+
It provides a pluggable backend architecture supporting popular options like RabbitMQ, alongside console and
|
|
14
14
|
debug backends for development. With features like a threaded change manager for non-blocking asynchronous
|
|
15
15
|
notifications, configurable connection retry mechanisms, and graceful shutdown handling, django-streaming ensures
|
|
16
16
|
reliable data delivery. It seamlessly integrates with Django's ORM via signals to automatically stream model changes
|
|
@@ -12,17 +12,17 @@ readme = "README.md"
|
|
|
12
12
|
authors = [
|
|
13
13
|
{ name = "Stefano Apostolico", email = "s.apostolico@gmail.com" },
|
|
14
14
|
]
|
|
15
|
-
requires-python = ">=3.
|
|
15
|
+
requires-python = ">=3.10"
|
|
16
16
|
classifiers = [
|
|
17
17
|
"Environment :: Web Environment",
|
|
18
18
|
"Framework :: Django",
|
|
19
|
+
"Framework :: Django :: 3.2",
|
|
19
20
|
"Framework :: Django :: 4.2",
|
|
20
21
|
"Framework :: Django :: 5.2",
|
|
21
22
|
"Intended Audience :: Developers",
|
|
22
23
|
"License :: OSI Approved :: MIT License",
|
|
23
24
|
"Operating System :: OS Independent",
|
|
24
25
|
"Programming Language :: Python :: 3 :: Only",
|
|
25
|
-
"Programming Language :: Python :: 3.9",
|
|
26
26
|
"Programming Language :: Python :: 3.10",
|
|
27
27
|
"Programming Language :: Python :: 3.11",
|
|
28
28
|
"Programming Language :: Python :: 3.12",
|
|
@@ -35,18 +35,13 @@ dynamic = [
|
|
|
35
35
|
]
|
|
36
36
|
dependencies = [
|
|
37
37
|
"click>=8",
|
|
38
|
-
"django>=
|
|
38
|
+
"django>=3",
|
|
39
39
|
"django-stubs-ext",
|
|
40
40
|
"markdown>=3.8.2",
|
|
41
|
-
"requests>=2.32.5",
|
|
42
|
-
]
|
|
43
|
-
optional-dependencies.rabbitmq = [
|
|
44
41
|
"pika>=1.3.2",
|
|
42
|
+
"requests>=2.32.5",
|
|
45
43
|
]
|
|
46
44
|
|
|
47
|
-
optional-dependencies.redis = [
|
|
48
|
-
"redis>=5",
|
|
49
|
-
]
|
|
50
45
|
urls.Documentation = "https://github.com/UNICEF/django-streaming"
|
|
51
46
|
urls.Homepage = "https://github.com/UNICEF/django-streaming"
|
|
52
47
|
|
|
@@ -94,7 +89,6 @@ type-check = [
|
|
|
94
89
|
"mypy",
|
|
95
90
|
"types-markdown",
|
|
96
91
|
"types-pika",
|
|
97
|
-
"types-redis",
|
|
98
92
|
"types-requests",
|
|
99
93
|
]
|
|
100
94
|
|
|
@@ -132,16 +132,16 @@ def listen(name: str, domain: str) -> None:
|
|
|
132
132
|
from streaming.manager import initialize_engine
|
|
133
133
|
|
|
134
134
|
manager = initialize_engine(True)
|
|
135
|
-
backend = manager.backend
|
|
135
|
+
backend: RabbitMQBackend = manager.backend # type: ignore[assignment]
|
|
136
136
|
|
|
137
|
-
if not isinstance(backend, RabbitMQBackend):
|
|
137
|
+
if not isinstance(manager.backend, RabbitMQBackend):
|
|
138
138
|
raise click.ClickException("RabbitMQ backend is not configured. Please set BROKER_URL to a rabbit:// URL.")
|
|
139
139
|
|
|
140
140
|
if not name:
|
|
141
141
|
name = random.choice(names) # noqa S311
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
142
|
+
|
|
143
|
+
backend.connection_name = name
|
|
144
|
+
backend.connect()
|
|
145
145
|
|
|
146
146
|
click.secho(f"Server: {backend.host}:{backend.port}")
|
|
147
147
|
click.secho(f"Consumer: {name}")
|
|
@@ -155,5 +155,4 @@ def listen(name: str, domain: str) -> None:
|
|
|
155
155
|
except KeyboardInterrupt:
|
|
156
156
|
click.secho("\nStopping listener.", fg="yellow")
|
|
157
157
|
finally:
|
|
158
|
-
|
|
159
|
-
backend.connection.close()
|
|
158
|
+
backend.close()
|
|
@@ -13,15 +13,15 @@ logger = logging.getLogger(__name__)
|
|
|
13
13
|
def get_backend() -> "BaseBackend":
|
|
14
14
|
from streaming.config import CONFIG
|
|
15
15
|
|
|
16
|
+
if not CONFIG.BROKER_URL:
|
|
17
|
+
raise StreamingConfigError("Empty BROKER_URL")
|
|
18
|
+
|
|
16
19
|
parsed_url = urlparse(CONFIG.BROKER_URL)
|
|
20
|
+
|
|
17
21
|
if parsed_url.scheme == "console":
|
|
18
22
|
from .console import ConsoleBackend
|
|
19
23
|
|
|
20
24
|
return ConsoleBackend(CONFIG.BROKER_URL)
|
|
21
|
-
if parsed_url.scheme == "redis":
|
|
22
|
-
from .redis import RedisBackend
|
|
23
|
-
|
|
24
|
-
return RedisBackend(CONFIG.BROKER_URL)
|
|
25
25
|
if parsed_url.scheme == "rabbit":
|
|
26
26
|
from .rabbitmq import RabbitMQBackend
|
|
27
27
|
|
|
@@ -30,4 +30,4 @@ def get_backend() -> "BaseBackend":
|
|
|
30
30
|
from .debug import DebugBackend
|
|
31
31
|
|
|
32
32
|
return DebugBackend(CONFIG.BROKER_URL)
|
|
33
|
-
raise StreamingConfigError(f"Broker not supported: {parsed_url.scheme}")
|
|
33
|
+
raise StreamingConfigError(f"Broker not supported: '{parsed_url.scheme}://{parsed_url.hostname}:{parsed_url.port}'")
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
from typing import TYPE_CHECKING
|
|
2
|
+
from typing import TYPE_CHECKING, Any
|
|
3
3
|
from urllib.parse import ParseResult, parse_qs, urlparse
|
|
4
4
|
|
|
5
5
|
from streaming.config import DEFAULT_QUEUE_NAME
|
|
@@ -22,5 +22,5 @@ class BaseBackend:
|
|
|
22
22
|
def initialize(self) -> None:
|
|
23
23
|
pass
|
|
24
24
|
|
|
25
|
-
def publish(self, message: "EventType") -> None:
|
|
25
|
+
def publish(self, message: "EventType", **kwargs: Any) -> None:
|
|
26
26
|
raise NotImplementedError()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import sys
|
|
3
|
-
from typing import TYPE_CHECKING
|
|
3
|
+
from typing import TYPE_CHECKING, Any
|
|
4
4
|
|
|
5
5
|
from ._base import BaseBackend
|
|
6
6
|
|
|
@@ -15,6 +15,9 @@ class ConsoleBackend(BaseBackend):
|
|
|
15
15
|
super().__init__(url)
|
|
16
16
|
self.stream = self._options.get("streams", "stdout")
|
|
17
17
|
|
|
18
|
-
def publish(self, message: "EventType") -> None:
|
|
18
|
+
def publish(self, message: "EventType", **kwargs: Any) -> None:
|
|
19
19
|
stream = getattr(sys, self.stream)
|
|
20
20
|
stream.write(f"{message}\n")
|
|
21
|
+
|
|
22
|
+
def listen(self, **kwargs: Any) -> None:
|
|
23
|
+
pass
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
from typing import TYPE_CHECKING
|
|
2
|
+
from typing import TYPE_CHECKING, Any
|
|
3
3
|
|
|
4
4
|
from ._base import BaseBackend
|
|
5
5
|
|
|
@@ -14,5 +14,5 @@ class DebugBackend(BaseBackend):
|
|
|
14
14
|
super().__init__(url)
|
|
15
15
|
self.messages: list[EventType] = []
|
|
16
16
|
|
|
17
|
-
def publish(self, message: "EventType") -> None:
|
|
17
|
+
def publish(self, message: "EventType", **kwargs: Any) -> None:
|
|
18
18
|
self.messages.append(message)
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
import atexit
|
|
2
|
+
import json
|
|
3
|
+
import logging
|
|
4
|
+
import socket
|
|
5
|
+
import time
|
|
6
|
+
from typing import TYPE_CHECKING, Any
|
|
7
|
+
|
|
8
|
+
import pika.channel
|
|
9
|
+
import pika.exceptions
|
|
10
|
+
from pika import PlainCredentials
|
|
11
|
+
from pika.exceptions import ConnectionClosedByBroker, ConnectionWrongStateError
|
|
12
|
+
from pika.exchange_type import ExchangeType
|
|
13
|
+
|
|
14
|
+
from streaming.config import CONFIG
|
|
15
|
+
|
|
16
|
+
from ..exceptions import StreamingCallbackError, StreamingCallbackFailure, StreamingConfigError
|
|
17
|
+
from ..utils import DAY, get_local_ip
|
|
18
|
+
from ._base import BaseBackend
|
|
19
|
+
|
|
20
|
+
if TYPE_CHECKING:
|
|
21
|
+
from pika.adapters.blocking_connection import BlockingChannel
|
|
22
|
+
from pika.spec import Basic, BasicProperties
|
|
23
|
+
|
|
24
|
+
from streaming.types import EventType, PikaCallback
|
|
25
|
+
|
|
26
|
+
logger = logging.getLogger(__name__)
|
|
27
|
+
|
|
28
|
+
MAX_RETRIES = 5
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class Callback:
|
|
32
|
+
def __init__(self, backend: "RabbitMQBackend", user_callback: "PikaCallback", ack: bool = True) -> None:
|
|
33
|
+
self.backend = backend
|
|
34
|
+
self.user_callback = user_callback
|
|
35
|
+
self.ack = ack
|
|
36
|
+
|
|
37
|
+
def __call__(
|
|
38
|
+
self, ch: "BlockingChannel", method: "Basic.Deliver", properties: "BasicProperties", body: bytes
|
|
39
|
+
) -> None:
|
|
40
|
+
try:
|
|
41
|
+
self.user_callback(ch, method, properties, body)
|
|
42
|
+
if self.ack:
|
|
43
|
+
ch.basic_ack(delivery_tag=method.delivery_tag) # type: ignore[arg-type]
|
|
44
|
+
except StreamingCallbackError as e:
|
|
45
|
+
evt: EventType = json.loads(body.decode())
|
|
46
|
+
retries = int(properties.headers.get("x-retries", 0)) # type: ignore[union-attr]
|
|
47
|
+
ch.basic_ack(method.delivery_tag) # type: ignore[arg-type]
|
|
48
|
+
self.backend._handle_retry(evt, ch, method, retries)
|
|
49
|
+
logger.debug("StreamingCallbackError", exc_info=e)
|
|
50
|
+
except StreamingCallbackFailure as e:
|
|
51
|
+
logger.error(f"Callback failure: {e}", exc_info=e)
|
|
52
|
+
except Exception as e:
|
|
53
|
+
logger.exception(f"Unexpected exception occurred: {e}", exc_info=e)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class RabbitMQBackend(BaseBackend):
|
|
57
|
+
def __init__(self, url: str) -> None:
|
|
58
|
+
super().__init__(url)
|
|
59
|
+
self.host = str(self._parsed_url.hostname)
|
|
60
|
+
self.port = int(self._parsed_url.port) if self._parsed_url.port else 5672
|
|
61
|
+
|
|
62
|
+
self.exchange = self._options.get("exchange", "django-streaming-broadcast")
|
|
63
|
+
self.retry_exchange = f"retry_{self.exchange}"
|
|
64
|
+
|
|
65
|
+
self.connection_name = self._options.get("connection_name", get_local_ip())
|
|
66
|
+
self.timeout = float(self._options.get("timeout", 0.5))
|
|
67
|
+
self.routing_key = self._options.get("routing_key", "")
|
|
68
|
+
self.virtual_host = self._options.get("virtual_host", "/")
|
|
69
|
+
|
|
70
|
+
self.connection: pika.BlockingConnection | None = None
|
|
71
|
+
self.channel: BlockingChannel | None = None
|
|
72
|
+
|
|
73
|
+
atexit.register(self.close)
|
|
74
|
+
|
|
75
|
+
def _configure(self) -> None:
|
|
76
|
+
if not self.connection:
|
|
77
|
+
raise StreamingConfigError("No active connection")
|
|
78
|
+
self.channel = self.connection.channel()
|
|
79
|
+
self.channel.exchange_declare(exchange=self.exchange, exchange_type=ExchangeType.topic, durable=True)
|
|
80
|
+
self.channel.exchange_declare(self.retry_exchange, exchange_type=ExchangeType.direct, durable=True)
|
|
81
|
+
|
|
82
|
+
self.channel.queue_declare(
|
|
83
|
+
queue=f"{self.exchange}_global_queue",
|
|
84
|
+
durable=True,
|
|
85
|
+
arguments={
|
|
86
|
+
"x-dead-letter-exchange": self.retry_exchange # failed → retry
|
|
87
|
+
},
|
|
88
|
+
)
|
|
89
|
+
self.channel.queue_bind(exchange=self.exchange, queue=f"{self.exchange}_global_queue", routing_key="#")
|
|
90
|
+
|
|
91
|
+
def _connect(self, raise_if_error: bool = False) -> None:
|
|
92
|
+
logger.debug("Connecting to %s:%s", self.host, self.port)
|
|
93
|
+
if self.connection and self.connection.is_open:
|
|
94
|
+
self.close()
|
|
95
|
+
|
|
96
|
+
for __ in range(CONFIG.RETRY_COUNT):
|
|
97
|
+
try:
|
|
98
|
+
if self._parsed_url.username:
|
|
99
|
+
auth = PlainCredentials(self._parsed_url.username, self._parsed_url.password or "")
|
|
100
|
+
else:
|
|
101
|
+
auth = PlainCredentials("guest", "guest")
|
|
102
|
+
self.connection = pika.BlockingConnection(
|
|
103
|
+
pika.ConnectionParameters(
|
|
104
|
+
host=self.host,
|
|
105
|
+
port=self.port,
|
|
106
|
+
virtual_host=self.virtual_host,
|
|
107
|
+
credentials=auth,
|
|
108
|
+
socket_timeout=self.timeout,
|
|
109
|
+
blocked_connection_timeout=self.timeout,
|
|
110
|
+
stack_timeout=self.timeout,
|
|
111
|
+
client_properties={
|
|
112
|
+
"connection_name": self.connection_name,
|
|
113
|
+
"product": "django-streaming",
|
|
114
|
+
"information": "",
|
|
115
|
+
"version": "1.0",
|
|
116
|
+
},
|
|
117
|
+
)
|
|
118
|
+
)
|
|
119
|
+
except (socket.gaierror, pika.exceptions.AMQPConnectionError) as e:
|
|
120
|
+
logger.warning(
|
|
121
|
+
f"Could not connect to RabbitMQ. Retrying in {CONFIG.RETRY_DELAY} seconds...",
|
|
122
|
+
)
|
|
123
|
+
time.sleep(CONFIG.RETRY_DELAY)
|
|
124
|
+
if raise_if_error:
|
|
125
|
+
raise StreamingConfigError(f"Error connecting {self.connection_url}") from e
|
|
126
|
+
logger.critical("Could not connect to RabbitMQ after multiple retries.")
|
|
127
|
+
|
|
128
|
+
def connect(self, raise_if_error: bool = False) -> None:
|
|
129
|
+
self._connect(raise_if_error)
|
|
130
|
+
if self.connection:
|
|
131
|
+
self._configure()
|
|
132
|
+
elif raise_if_error:
|
|
133
|
+
raise StreamingConfigError("No active connection")
|
|
134
|
+
|
|
135
|
+
def _basic_publish(self, message: "EventType", retry_count: int = 0) -> None:
|
|
136
|
+
if not self.channel:
|
|
137
|
+
raise StreamingConfigError("No active channel")
|
|
138
|
+
self.channel.basic_publish(
|
|
139
|
+
exchange=self.exchange,
|
|
140
|
+
routing_key=message.get("domain", self.routing_key) or self.routing_key,
|
|
141
|
+
body=json.dumps(message).encode(),
|
|
142
|
+
properties=pika.BasicProperties(
|
|
143
|
+
delivery_mode=pika.spec.PERSISTENT_DELIVERY_MODE,
|
|
144
|
+
expiration=str(DAY * 2 * 1000), # milliseconds
|
|
145
|
+
headers={"x-retries": retry_count},
|
|
146
|
+
),
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
def _handle_retry(self, message: "EventType", ch: "BlockingChannel", method: "Basic.Deliver", retries: int) -> None:
|
|
150
|
+
ch.basic_ack(method.delivery_tag) # type: ignore[arg-type]
|
|
151
|
+
if retries < MAX_RETRIES:
|
|
152
|
+
delay = 2000 * (2**retries) # ms (exponential backoff)
|
|
153
|
+
delay_queue = f"{self.exchange}_retry_{delay}ms"
|
|
154
|
+
|
|
155
|
+
# Declare a delay queue with TTL
|
|
156
|
+
ch.queue_declare(
|
|
157
|
+
delay_queue,
|
|
158
|
+
durable=True,
|
|
159
|
+
arguments={
|
|
160
|
+
"x-dead-letter-exchange": self.exchange, # after delay → back to main
|
|
161
|
+
"x-message-ttl": delay,
|
|
162
|
+
},
|
|
163
|
+
)
|
|
164
|
+
ch.queue_bind(delay_queue, self.retry_exchange, routing_key="task")
|
|
165
|
+
self._basic_publish(message, retries + 1)
|
|
166
|
+
else:
|
|
167
|
+
logger.error(f"Dropping after {MAX_RETRIES} retries")
|
|
168
|
+
|
|
169
|
+
def listen(self, domains: list[str], callback: "PikaCallback", ack: bool = True) -> None:
|
|
170
|
+
if self.channel is None:
|
|
171
|
+
self.connect()
|
|
172
|
+
_callback = Callback(self, callback, ack=ack)
|
|
173
|
+
for domain in domains:
|
|
174
|
+
queue_name = f"{self.connection_name.lower()}_sub_to_{domain}"
|
|
175
|
+
self.channel.queue_declare(queue=queue_name, durable=True) # type: ignore[union-attr]
|
|
176
|
+
self.channel.queue_bind(exchange=self.exchange, queue=queue_name, routing_key=domain) # type: ignore[union-attr]
|
|
177
|
+
self.channel.basic_consume(queue=queue_name, on_message_callback=_callback, auto_ack=False) # type: ignore[union-attr]
|
|
178
|
+
self.channel.start_consuming() # type: ignore[union-attr]
|
|
179
|
+
|
|
180
|
+
def publish(self, message: "EventType", retry_count: int = 0, **kwargs: Any) -> None:
|
|
181
|
+
if not self.channel or self.channel.is_closed:
|
|
182
|
+
self.connect()
|
|
183
|
+
if self.channel:
|
|
184
|
+
logger.debug("publish to %s %s", self.exchange, message.get("domain", ""))
|
|
185
|
+
try:
|
|
186
|
+
self._basic_publish(message, 0)
|
|
187
|
+
except Exception as e: # noqa: BLE001
|
|
188
|
+
logger.critical("Unhandled error sending to RabbitMQ. Message not published.", exc_info=e)
|
|
189
|
+
else:
|
|
190
|
+
logger.critical("RabbitMQ connection not available after reconnect. Message not published.")
|
|
191
|
+
|
|
192
|
+
def close(self) -> None:
|
|
193
|
+
logger.debug("Closing RabbitMQ connection.")
|
|
194
|
+
try:
|
|
195
|
+
self.connection.close() # type: ignore[union-attr]
|
|
196
|
+
except (ConnectionClosedByBroker, AttributeError, ConnectionWrongStateError):
|
|
197
|
+
pass
|
|
198
|
+
finally:
|
|
199
|
+
self.connection = None
|
|
200
|
+
self.channel = None
|
|
@@ -13,7 +13,7 @@ DEFAULT_QUEUE_NAME = "django_model_changes"
|
|
|
13
13
|
|
|
14
14
|
class StreamingConfig:
|
|
15
15
|
_DEFAULTS = {
|
|
16
|
-
"BROKER_URL": "
|
|
16
|
+
"BROKER_URL": "console://",
|
|
17
17
|
"RETRY_COUNT": 3,
|
|
18
18
|
"RETRY_DELAY": 1,
|
|
19
19
|
"MANAGER_CLASS": "streaming.manager.ChangeManager",
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
logger = logging.getLogger(__name__)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class StreamingError(Exception):
|
|
7
|
+
pass
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class StreamingConfigError(StreamingError):
|
|
11
|
+
pass
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class StreamingBackendError(StreamingError):
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class StreamingCallbackError(StreamingError):
|
|
19
|
+
"""Generic exception raised when a callback fails. Message will be re-queued."""
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class StreamingCallbackFailure(StreamingError): # noqa: N818
|
|
23
|
+
"""Generic exception raised when a callback fails. Message will be discarded."""
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import socket
|
|
1
2
|
from typing import TYPE_CHECKING, Any
|
|
2
3
|
|
|
3
4
|
if TYPE_CHECKING:
|
|
@@ -20,3 +21,10 @@ def make_event(message: "str | JSON", *, event: str = "", domain: str = "") -> "
|
|
|
20
21
|
else:
|
|
21
22
|
payload = message
|
|
22
23
|
return {"event": event, "domain": domain, "payload": payload}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def get_local_ip() -> str:
|
|
27
|
+
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
28
|
+
s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
|
|
29
|
+
s.connect(("<broadcast>", 12345)) # 12345 is random port. 0 fails on Mac.
|
|
30
|
+
return str(s.getsockname()[0])
|
|
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
|
|
|
28
28
|
commit_id: COMMIT_ID
|
|
29
29
|
__commit_id__: COMMIT_ID
|
|
30
30
|
|
|
31
|
-
__version__ = version = '0.
|
|
32
|
-
__version_tuple__ = version_tuple = (0,
|
|
31
|
+
__version__ = version = '0.4'
|
|
32
|
+
__version_tuple__ = version_tuple = (0, 4)
|
|
33
33
|
|
|
34
34
|
__commit_id__ = commit_id = None
|
|
@@ -1,127 +0,0 @@
|
|
|
1
|
-
import atexit
|
|
2
|
-
import json
|
|
3
|
-
import logging
|
|
4
|
-
import socket
|
|
5
|
-
import time
|
|
6
|
-
from typing import TYPE_CHECKING
|
|
7
|
-
|
|
8
|
-
import pika.channel
|
|
9
|
-
import pika.exceptions
|
|
10
|
-
from pika import PlainCredentials
|
|
11
|
-
from pika.exchange_type import ExchangeType
|
|
12
|
-
|
|
13
|
-
from streaming.config import CONFIG
|
|
14
|
-
|
|
15
|
-
from ..utils import DAY
|
|
16
|
-
from ._base import BaseBackend
|
|
17
|
-
|
|
18
|
-
if TYPE_CHECKING:
|
|
19
|
-
from pika.adapters.blocking_connection import BlockingChannel
|
|
20
|
-
from pika.spec import Basic, BasicProperties
|
|
21
|
-
|
|
22
|
-
from streaming.types import EventType, PikaCallback
|
|
23
|
-
|
|
24
|
-
logger = logging.getLogger(__name__)
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
class RabbitMQBackend(BaseBackend):
|
|
28
|
-
def __init__(self, url: str) -> None:
|
|
29
|
-
super().__init__(url)
|
|
30
|
-
self.host = str(self._parsed_url.hostname)
|
|
31
|
-
self.port = int(self._parsed_url.port) if self._parsed_url.port else 5672
|
|
32
|
-
|
|
33
|
-
self.exchange = self._options.get("exchange", "django-streaming-broadcast")
|
|
34
|
-
self.connection_name = self._options.get("connection_name", "")
|
|
35
|
-
self.timeout = float(self._options.get("timeout", 0.5))
|
|
36
|
-
self.routing_key = self._options.get("routing_key", "")
|
|
37
|
-
self.virtual_host = self._options.get("virtual_host", "/")
|
|
38
|
-
|
|
39
|
-
self.connection: pika.BlockingConnection | None = None
|
|
40
|
-
self.channel: BlockingChannel | None = None
|
|
41
|
-
|
|
42
|
-
atexit.register(self.close)
|
|
43
|
-
|
|
44
|
-
def connect(self) -> None:
|
|
45
|
-
logger.debug("Connecting to %s:%s", self.host, self.port)
|
|
46
|
-
if self.connection and self.connection.is_open:
|
|
47
|
-
self.close()
|
|
48
|
-
|
|
49
|
-
for __ in range(CONFIG.RETRY_COUNT):
|
|
50
|
-
try:
|
|
51
|
-
if self._parsed_url.username:
|
|
52
|
-
auth = PlainCredentials(self._parsed_url.username, self._parsed_url.password or "")
|
|
53
|
-
else:
|
|
54
|
-
auth = PlainCredentials("guest", "guest")
|
|
55
|
-
self.connection = pika.BlockingConnection(
|
|
56
|
-
pika.ConnectionParameters(
|
|
57
|
-
host=self.host,
|
|
58
|
-
port=self.port,
|
|
59
|
-
virtual_host=self.virtual_host,
|
|
60
|
-
credentials=auth,
|
|
61
|
-
socket_timeout=self.timeout,
|
|
62
|
-
blocked_connection_timeout=self.timeout,
|
|
63
|
-
stack_timeout=self.timeout,
|
|
64
|
-
client_properties={
|
|
65
|
-
"connection_name": self.connection_name,
|
|
66
|
-
"product": "django-streaming",
|
|
67
|
-
"information": "",
|
|
68
|
-
"version": "1.0",
|
|
69
|
-
},
|
|
70
|
-
)
|
|
71
|
-
)
|
|
72
|
-
self.channel = self.connection.channel()
|
|
73
|
-
self.channel.exchange_declare(exchange=self.exchange, exchange_type=ExchangeType.topic, durable=True)
|
|
74
|
-
self.channel.queue_declare(queue="global_queue", durable=True)
|
|
75
|
-
self.channel.queue_bind(exchange=self.exchange, queue="global_queue", routing_key="#")
|
|
76
|
-
return
|
|
77
|
-
except (socket.gaierror, pika.exceptions.AMQPConnectionError):
|
|
78
|
-
logger.warning(
|
|
79
|
-
f"Could not connect to RabbitMQ. Retrying in {CONFIG.RETRY_DELAY} seconds...",
|
|
80
|
-
)
|
|
81
|
-
time.sleep(CONFIG.RETRY_DELAY)
|
|
82
|
-
logger.critical("Could not connect to RabbitMQ after multiple retries.")
|
|
83
|
-
|
|
84
|
-
def listen(self, domains: list[str], callback: "PikaCallback", ack: bool = True) -> None:
|
|
85
|
-
def _callback(
|
|
86
|
-
ch: "BlockingChannel", method: "Basic.Deliver", properties: "BasicProperties", body: bytes
|
|
87
|
-
) -> None:
|
|
88
|
-
callback(ch, method, properties, body)
|
|
89
|
-
if ack:
|
|
90
|
-
ch.basic_ack(delivery_tag=method.delivery_tag) # type: ignore[arg-type]
|
|
91
|
-
|
|
92
|
-
if self.channel is None:
|
|
93
|
-
self.connect()
|
|
94
|
-
|
|
95
|
-
for domain in domains:
|
|
96
|
-
queue_name = f"{self.connection_name.lower()}_sub_to_{domain}"
|
|
97
|
-
self.channel.queue_declare(queue=queue_name, durable=True) # type: ignore[union-attr]
|
|
98
|
-
self.channel.queue_bind(exchange=self.exchange, queue=queue_name, routing_key=domain) # type: ignore[union-attr]
|
|
99
|
-
self.channel.basic_consume(queue=queue_name, on_message_callback=_callback, auto_ack=False) # type: ignore[union-attr]
|
|
100
|
-
self.channel.start_consuming() # type: ignore[union-attr]
|
|
101
|
-
|
|
102
|
-
def publish(self, message: "EventType") -> None:
|
|
103
|
-
if not self.channel or self.channel.is_closed:
|
|
104
|
-
self.connect()
|
|
105
|
-
if self.channel:
|
|
106
|
-
logger.debug("publish to %s %s", self.exchange, message.get("domain", ""))
|
|
107
|
-
try:
|
|
108
|
-
self.channel.basic_publish(
|
|
109
|
-
exchange=self.exchange,
|
|
110
|
-
routing_key=message.get("domain", self.routing_key) or self.routing_key,
|
|
111
|
-
body=json.dumps(message).encode(),
|
|
112
|
-
properties=pika.BasicProperties(
|
|
113
|
-
delivery_mode=pika.spec.PERSISTENT_DELIVERY_MODE,
|
|
114
|
-
expiration=str(DAY * 2 * 1000), # milliseconds
|
|
115
|
-
),
|
|
116
|
-
)
|
|
117
|
-
except Exception as e: # noqa: BLE001
|
|
118
|
-
logger.critical("Unhandled error sending to RabbitMQ. Message not published.", exc_info=e)
|
|
119
|
-
else:
|
|
120
|
-
logger.critical("RabbitMQ connection not available after reconnect. Message not published.")
|
|
121
|
-
|
|
122
|
-
def close(self) -> None:
|
|
123
|
-
if self.connection and self.connection.is_open:
|
|
124
|
-
logger.debug("Closing RabbitMQ connection.")
|
|
125
|
-
self.connection.close()
|
|
126
|
-
self.connection = None
|
|
127
|
-
self.channel = None
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
import json
|
|
2
|
-
import logging
|
|
3
|
-
import time
|
|
4
|
-
from typing import TYPE_CHECKING
|
|
5
|
-
|
|
6
|
-
import redis
|
|
7
|
-
from redis.exceptions import ConnectionError as RedisConnectionError
|
|
8
|
-
|
|
9
|
-
from streaming.config import CONFIG
|
|
10
|
-
from streaming.exceptions import StreamingBackendError
|
|
11
|
-
|
|
12
|
-
from ._base import BaseBackend
|
|
13
|
-
|
|
14
|
-
if TYPE_CHECKING:
|
|
15
|
-
from redis import Redis
|
|
16
|
-
|
|
17
|
-
from ..types import EventType
|
|
18
|
-
|
|
19
|
-
logger = logging.getLogger(__name__)
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
class RedisBackend(BaseBackend):
|
|
23
|
-
redis_client: "Redis[bytes]"
|
|
24
|
-
|
|
25
|
-
def __init__(self, url: str) -> None:
|
|
26
|
-
super().__init__(url)
|
|
27
|
-
self.db = int(self._parsed_url.path.lstrip("/") or 0)
|
|
28
|
-
self.host = str(self._parsed_url.hostname)
|
|
29
|
-
self.port = int(self._parsed_url.port) if self._parsed_url.port else 6379
|
|
30
|
-
self.timeout = float(self._options.get("timeout", 0.5))
|
|
31
|
-
|
|
32
|
-
self.redis_client = self._get_client()
|
|
33
|
-
|
|
34
|
-
def _get_client(self) -> "Redis[bytes]":
|
|
35
|
-
for __ in range(CONFIG.RETRY_COUNT):
|
|
36
|
-
try:
|
|
37
|
-
client = redis.Redis(
|
|
38
|
-
host=self.host,
|
|
39
|
-
port=self.port,
|
|
40
|
-
db=self.db,
|
|
41
|
-
socket_timeout=self.timeout,
|
|
42
|
-
socket_connect_timeout=self.timeout,
|
|
43
|
-
)
|
|
44
|
-
client.ping()
|
|
45
|
-
return client
|
|
46
|
-
except RedisConnectionError:
|
|
47
|
-
logger.warning("Could not connect to Redis. Retrying in %s seconds...", CONFIG.RETRY_DELAY)
|
|
48
|
-
time.sleep(CONFIG.RETRY_DELAY)
|
|
49
|
-
raise StreamingBackendError("Could not connect to Redis after multiple retries.")
|
|
50
|
-
|
|
51
|
-
def publish(self, message: "EventType") -> None:
|
|
52
|
-
try:
|
|
53
|
-
self.redis_client.publish(self.queue_name, json.dumps(message).encode())
|
|
54
|
-
except RedisConnectionError:
|
|
55
|
-
self.redis_client = self._get_client()
|
|
56
|
-
self.redis_client.publish(self.queue_name, json.dumps(message).encode())
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|