django-streaming 0.2__tar.gz → 0.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (24) hide show
  1. {django_streaming-0.2 → django_streaming-0.4}/PKG-INFO +6 -9
  2. {django_streaming-0.2 → django_streaming-0.4}/README.md +1 -1
  3. {django_streaming-0.2 → django_streaming-0.4}/pyproject.toml +4 -10
  4. {django_streaming-0.2 → django_streaming-0.4}/src/streaming/__cli__.py +6 -7
  5. {django_streaming-0.2 → django_streaming-0.4}/src/streaming/apps.py +0 -5
  6. {django_streaming-0.2 → django_streaming-0.4}/src/streaming/backends/__init__.py +5 -5
  7. {django_streaming-0.2 → django_streaming-0.4}/src/streaming/backends/_base.py +2 -2
  8. {django_streaming-0.2 → django_streaming-0.4}/src/streaming/backends/console.py +5 -2
  9. {django_streaming-0.2 → django_streaming-0.4}/src/streaming/backends/debug.py +2 -2
  10. django_streaming-0.4/src/streaming/backends/rabbitmq.py +200 -0
  11. {django_streaming-0.2 → django_streaming-0.4}/src/streaming/config.py +1 -1
  12. django_streaming-0.4/src/streaming/exceptions.py +23 -0
  13. {django_streaming-0.2 → django_streaming-0.4}/src/streaming/utils.py +8 -0
  14. {django_streaming-0.2 → django_streaming-0.4}/src/streaming/version.py +2 -2
  15. django_streaming-0.2/src/streaming/backends/rabbitmq.py +0 -126
  16. django_streaming-0.2/src/streaming/backends/redis.py +0 -56
  17. django_streaming-0.2/src/streaming/exceptions.py +0 -15
  18. {django_streaming-0.2 → django_streaming-0.4}/.gitignore +0 -0
  19. {django_streaming-0.2 → django_streaming-0.4}/LICENSE +0 -0
  20. {django_streaming-0.2 → django_streaming-0.4}/src/streaming/api.py +0 -0
  21. {django_streaming-0.2 → django_streaming-0.4}/src/streaming/handlers.py +0 -0
  22. {django_streaming-0.2 → django_streaming-0.4}/src/streaming/manager.py +0 -0
  23. {django_streaming-0.2 → django_streaming-0.4}/src/streaming/py.typed +0 -0
  24. {django_streaming-0.2 → django_streaming-0.4}/src/streaming/types.pyi +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: django-streaming
3
- Version: 0.2
3
+ Version: 0.4
4
4
  Summary: A project for streaming data.
5
5
  Project-URL: Documentation, https://github.com/UNICEF/django-streaming
6
6
  Project-URL: Homepage, https://github.com/UNICEF/django-streaming
@@ -8,29 +8,26 @@ Author-email: Stefano Apostolico <s.apostolico@gmail.com>
8
8
  License-File: LICENSE
9
9
  Classifier: Environment :: Web Environment
10
10
  Classifier: Framework :: Django
11
+ Classifier: Framework :: Django :: 3.2
11
12
  Classifier: Framework :: Django :: 4.2
12
13
  Classifier: Framework :: Django :: 5.2
13
14
  Classifier: Intended Audience :: Developers
14
15
  Classifier: License :: OSI Approved :: MIT License
15
16
  Classifier: Operating System :: OS Independent
16
17
  Classifier: Programming Language :: Python :: 3 :: Only
17
- Classifier: Programming Language :: Python :: 3.9
18
18
  Classifier: Programming Language :: Python :: 3.10
19
19
  Classifier: Programming Language :: Python :: 3.11
20
20
  Classifier: Programming Language :: Python :: 3.12
21
21
  Classifier: Programming Language :: Python :: 3.13
22
22
  Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
23
23
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
24
- Requires-Python: >=3.9
24
+ Requires-Python: >=3.10
25
25
  Requires-Dist: click>=8
26
26
  Requires-Dist: django-stubs-ext
27
- Requires-Dist: django>=4
27
+ Requires-Dist: django>=3
28
28
  Requires-Dist: markdown>=3.8.2
29
+ Requires-Dist: pika>=1.3.2
29
30
  Requires-Dist: requests>=2.32.5
30
- Provides-Extra: rabbitmq
31
- Requires-Dist: pika>=1.3.2; extra == 'rabbitmq'
32
- Provides-Extra: redis
33
- Requires-Dist: redis>=5; extra == 'redis'
34
31
  Description-Content-Type: text/markdown
35
32
 
36
33
  # Django Streaming
@@ -45,7 +42,7 @@ Description-Content-Type: text/markdown
45
42
  `django-streaming` is a robust and flexible Django application designed for efficiently streaming data from your
46
43
  Django projects to various message brokers.
47
44
 
48
- It provides a pluggable backend architecture supporting popular options like RabbitMQ and Redis, alongside console and
45
+ It provides a pluggable backend architecture supporting popular options like RabbitMQ, alongside console and
49
46
  debug backends for development. With features like a threaded change manager for non-blocking asynchronous
50
47
  notifications, configurable connection retry mechanisms, and graceful shutdown handling, django-streaming ensures
51
48
  reliable data delivery. It seamlessly integrates with Django's ORM via signals to automatically stream model changes
@@ -10,7 +10,7 @@
10
10
  `django-streaming` is a robust and flexible Django application designed for efficiently streaming data from your
11
11
  Django projects to various message brokers.
12
12
 
13
- It provides a pluggable backend architecture supporting popular options like RabbitMQ and Redis, alongside console and
13
+ It provides a pluggable backend architecture supporting popular options like RabbitMQ, alongside console and
14
14
  debug backends for development. With features like a threaded change manager for non-blocking asynchronous
15
15
  notifications, configurable connection retry mechanisms, and graceful shutdown handling, django-streaming ensures
16
16
  reliable data delivery. It seamlessly integrates with Django's ORM via signals to automatically stream model changes
@@ -12,17 +12,17 @@ readme = "README.md"
12
12
  authors = [
13
13
  { name = "Stefano Apostolico", email = "s.apostolico@gmail.com" },
14
14
  ]
15
- requires-python = ">=3.9"
15
+ requires-python = ">=3.10"
16
16
  classifiers = [
17
17
  "Environment :: Web Environment",
18
18
  "Framework :: Django",
19
+ "Framework :: Django :: 3.2",
19
20
  "Framework :: Django :: 4.2",
20
21
  "Framework :: Django :: 5.2",
21
22
  "Intended Audience :: Developers",
22
23
  "License :: OSI Approved :: MIT License",
23
24
  "Operating System :: OS Independent",
24
25
  "Programming Language :: Python :: 3 :: Only",
25
- "Programming Language :: Python :: 3.9",
26
26
  "Programming Language :: Python :: 3.10",
27
27
  "Programming Language :: Python :: 3.11",
28
28
  "Programming Language :: Python :: 3.12",
@@ -35,18 +35,13 @@ dynamic = [
35
35
  ]
36
36
  dependencies = [
37
37
  "click>=8",
38
- "django>=4",
38
+ "django>=3",
39
39
  "django-stubs-ext",
40
40
  "markdown>=3.8.2",
41
- "requests>=2.32.5",
42
- ]
43
- optional-dependencies.rabbitmq = [
44
41
  "pika>=1.3.2",
42
+ "requests>=2.32.5",
45
43
  ]
46
44
 
47
- optional-dependencies.redis = [
48
- "redis>=5",
49
- ]
50
45
  urls.Documentation = "https://github.com/UNICEF/django-streaming"
51
46
  urls.Homepage = "https://github.com/UNICEF/django-streaming"
52
47
 
@@ -94,7 +89,6 @@ type-check = [
94
89
  "mypy",
95
90
  "types-markdown",
96
91
  "types-pika",
97
- "types-redis",
98
92
  "types-requests",
99
93
  ]
100
94
 
@@ -132,16 +132,16 @@ def listen(name: str, domain: str) -> None:
132
132
  from streaming.manager import initialize_engine
133
133
 
134
134
  manager = initialize_engine(True)
135
- backend = manager.backend
135
+ backend: RabbitMQBackend = manager.backend # type: ignore[assignment]
136
136
 
137
- if not isinstance(backend, RabbitMQBackend):
137
+ if not isinstance(manager.backend, RabbitMQBackend):
138
138
  raise click.ClickException("RabbitMQ backend is not configured. Please set BROKER_URL to a rabbit:// URL.")
139
139
 
140
140
  if not name:
141
141
  name = random.choice(names) # noqa S311
142
- if name != backend.connection_name:
143
- backend.connection_name = name
144
- backend.connect()
142
+
143
+ backend.connection_name = name
144
+ backend.connect()
145
145
 
146
146
  click.secho(f"Server: {backend.host}:{backend.port}")
147
147
  click.secho(f"Consumer: {name}")
@@ -155,5 +155,4 @@ def listen(name: str, domain: str) -> None:
155
155
  except KeyboardInterrupt:
156
156
  click.secho("\nStopping listener.", fg="yellow")
157
157
  finally:
158
- if backend.connection and backend.connection.is_open:
159
- backend.connection.close()
158
+ backend.close()
@@ -10,8 +10,3 @@ class StreamingConfig(AppConfig):
10
10
  default_auto_field = "django.db.models.BigAutoField"
11
11
  name = "streaming"
12
12
  path = str(Path(__file__).parent)
13
-
14
- def ready(self) -> None:
15
- from .manager import initialize_engine
16
-
17
- initialize_engine()
@@ -13,15 +13,15 @@ logger = logging.getLogger(__name__)
13
13
  def get_backend() -> "BaseBackend":
14
14
  from streaming.config import CONFIG
15
15
 
16
+ if not CONFIG.BROKER_URL:
17
+ raise StreamingConfigError("Empty BROKER_URL")
18
+
16
19
  parsed_url = urlparse(CONFIG.BROKER_URL)
20
+
17
21
  if parsed_url.scheme == "console":
18
22
  from .console import ConsoleBackend
19
23
 
20
24
  return ConsoleBackend(CONFIG.BROKER_URL)
21
- if parsed_url.scheme == "redis":
22
- from .redis import RedisBackend
23
-
24
- return RedisBackend(CONFIG.BROKER_URL)
25
25
  if parsed_url.scheme == "rabbit":
26
26
  from .rabbitmq import RabbitMQBackend
27
27
 
@@ -30,4 +30,4 @@ def get_backend() -> "BaseBackend":
30
30
  from .debug import DebugBackend
31
31
 
32
32
  return DebugBackend(CONFIG.BROKER_URL)
33
- raise StreamingConfigError(f"Broker not supported: {parsed_url.scheme}")
33
+ raise StreamingConfigError(f"Broker not supported: '{parsed_url.scheme}://{parsed_url.hostname}:{parsed_url.port}'")
@@ -1,5 +1,5 @@
1
1
  import logging
2
- from typing import TYPE_CHECKING
2
+ from typing import TYPE_CHECKING, Any
3
3
  from urllib.parse import ParseResult, parse_qs, urlparse
4
4
 
5
5
  from streaming.config import DEFAULT_QUEUE_NAME
@@ -22,5 +22,5 @@ class BaseBackend:
22
22
  def initialize(self) -> None:
23
23
  pass
24
24
 
25
- def publish(self, message: "EventType") -> None:
25
+ def publish(self, message: "EventType", **kwargs: Any) -> None:
26
26
  raise NotImplementedError()
@@ -1,6 +1,6 @@
1
1
  import logging
2
2
  import sys
3
- from typing import TYPE_CHECKING
3
+ from typing import TYPE_CHECKING, Any
4
4
 
5
5
  from ._base import BaseBackend
6
6
 
@@ -15,6 +15,9 @@ class ConsoleBackend(BaseBackend):
15
15
  super().__init__(url)
16
16
  self.stream = self._options.get("streams", "stdout")
17
17
 
18
- def publish(self, message: "EventType") -> None:
18
+ def publish(self, message: "EventType", **kwargs: Any) -> None:
19
19
  stream = getattr(sys, self.stream)
20
20
  stream.write(f"{message}\n")
21
+
22
+ def listen(self, **kwargs: Any) -> None:
23
+ pass
@@ -1,5 +1,5 @@
1
1
  import logging
2
- from typing import TYPE_CHECKING
2
+ from typing import TYPE_CHECKING, Any
3
3
 
4
4
  from ._base import BaseBackend
5
5
 
@@ -14,5 +14,5 @@ class DebugBackend(BaseBackend):
14
14
  super().__init__(url)
15
15
  self.messages: list[EventType] = []
16
16
 
17
- def publish(self, message: "EventType") -> None:
17
+ def publish(self, message: "EventType", **kwargs: Any) -> None:
18
18
  self.messages.append(message)
@@ -0,0 +1,200 @@
1
+ import atexit
2
+ import json
3
+ import logging
4
+ import socket
5
+ import time
6
+ from typing import TYPE_CHECKING, Any
7
+
8
+ import pika.channel
9
+ import pika.exceptions
10
+ from pika import PlainCredentials
11
+ from pika.exceptions import ConnectionClosedByBroker, ConnectionWrongStateError
12
+ from pika.exchange_type import ExchangeType
13
+
14
+ from streaming.config import CONFIG
15
+
16
+ from ..exceptions import StreamingCallbackError, StreamingCallbackFailure, StreamingConfigError
17
+ from ..utils import DAY, get_local_ip
18
+ from ._base import BaseBackend
19
+
20
+ if TYPE_CHECKING:
21
+ from pika.adapters.blocking_connection import BlockingChannel
22
+ from pika.spec import Basic, BasicProperties
23
+
24
+ from streaming.types import EventType, PikaCallback
25
+
26
+ logger = logging.getLogger(__name__)
27
+
28
+ MAX_RETRIES = 5
29
+
30
+
31
+ class Callback:
32
+ def __init__(self, backend: "RabbitMQBackend", user_callback: "PikaCallback", ack: bool = True) -> None:
33
+ self.backend = backend
34
+ self.user_callback = user_callback
35
+ self.ack = ack
36
+
37
+ def __call__(
38
+ self, ch: "BlockingChannel", method: "Basic.Deliver", properties: "BasicProperties", body: bytes
39
+ ) -> None:
40
+ try:
41
+ self.user_callback(ch, method, properties, body)
42
+ if self.ack:
43
+ ch.basic_ack(delivery_tag=method.delivery_tag) # type: ignore[arg-type]
44
+ except StreamingCallbackError as e:
45
+ evt: EventType = json.loads(body.decode())
46
+ retries = int(properties.headers.get("x-retries", 0)) # type: ignore[union-attr]
47
+ ch.basic_ack(method.delivery_tag) # type: ignore[arg-type]
48
+ self.backend._handle_retry(evt, ch, method, retries)
49
+ logger.debug("StreamingCallbackError", exc_info=e)
50
+ except StreamingCallbackFailure as e:
51
+ logger.error(f"Callback failure: {e}", exc_info=e)
52
+ except Exception as e:
53
+ logger.exception(f"Unexpected exception occurred: {e}", exc_info=e)
54
+
55
+
56
+ class RabbitMQBackend(BaseBackend):
57
+ def __init__(self, url: str) -> None:
58
+ super().__init__(url)
59
+ self.host = str(self._parsed_url.hostname)
60
+ self.port = int(self._parsed_url.port) if self._parsed_url.port else 5672
61
+
62
+ self.exchange = self._options.get("exchange", "django-streaming-broadcast")
63
+ self.retry_exchange = f"retry_{self.exchange}"
64
+
65
+ self.connection_name = self._options.get("connection_name", get_local_ip())
66
+ self.timeout = float(self._options.get("timeout", 0.5))
67
+ self.routing_key = self._options.get("routing_key", "")
68
+ self.virtual_host = self._options.get("virtual_host", "/")
69
+
70
+ self.connection: pika.BlockingConnection | None = None
71
+ self.channel: BlockingChannel | None = None
72
+
73
+ atexit.register(self.close)
74
+
75
+ def _configure(self) -> None:
76
+ if not self.connection:
77
+ raise StreamingConfigError("No active connection")
78
+ self.channel = self.connection.channel()
79
+ self.channel.exchange_declare(exchange=self.exchange, exchange_type=ExchangeType.topic, durable=True)
80
+ self.channel.exchange_declare(self.retry_exchange, exchange_type=ExchangeType.direct, durable=True)
81
+
82
+ self.channel.queue_declare(
83
+ queue=f"{self.exchange}_global_queue",
84
+ durable=True,
85
+ arguments={
86
+ "x-dead-letter-exchange": self.retry_exchange # failed → retry
87
+ },
88
+ )
89
+ self.channel.queue_bind(exchange=self.exchange, queue=f"{self.exchange}_global_queue", routing_key="#")
90
+
91
+ def _connect(self, raise_if_error: bool = False) -> None:
92
+ logger.debug("Connecting to %s:%s", self.host, self.port)
93
+ if self.connection and self.connection.is_open:
94
+ self.close()
95
+
96
+ for __ in range(CONFIG.RETRY_COUNT):
97
+ try:
98
+ if self._parsed_url.username:
99
+ auth = PlainCredentials(self._parsed_url.username, self._parsed_url.password or "")
100
+ else:
101
+ auth = PlainCredentials("guest", "guest")
102
+ self.connection = pika.BlockingConnection(
103
+ pika.ConnectionParameters(
104
+ host=self.host,
105
+ port=self.port,
106
+ virtual_host=self.virtual_host,
107
+ credentials=auth,
108
+ socket_timeout=self.timeout,
109
+ blocked_connection_timeout=self.timeout,
110
+ stack_timeout=self.timeout,
111
+ client_properties={
112
+ "connection_name": self.connection_name,
113
+ "product": "django-streaming",
114
+ "information": "",
115
+ "version": "1.0",
116
+ },
117
+ )
118
+ )
119
+ except (socket.gaierror, pika.exceptions.AMQPConnectionError) as e:
120
+ logger.warning(
121
+ f"Could not connect to RabbitMQ. Retrying in {CONFIG.RETRY_DELAY} seconds...",
122
+ )
123
+ time.sleep(CONFIG.RETRY_DELAY)
124
+ if raise_if_error:
125
+ raise StreamingConfigError(f"Error connecting {self.connection_url}") from e
126
+ logger.critical("Could not connect to RabbitMQ after multiple retries.")
127
+
128
+ def connect(self, raise_if_error: bool = False) -> None:
129
+ self._connect(raise_if_error)
130
+ if self.connection:
131
+ self._configure()
132
+ elif raise_if_error:
133
+ raise StreamingConfigError("No active connection")
134
+
135
+ def _basic_publish(self, message: "EventType", retry_count: int = 0) -> None:
136
+ if not self.channel:
137
+ raise StreamingConfigError("No active channel")
138
+ self.channel.basic_publish(
139
+ exchange=self.exchange,
140
+ routing_key=message.get("domain", self.routing_key) or self.routing_key,
141
+ body=json.dumps(message).encode(),
142
+ properties=pika.BasicProperties(
143
+ delivery_mode=pika.spec.PERSISTENT_DELIVERY_MODE,
144
+ expiration=str(DAY * 2 * 1000), # milliseconds
145
+ headers={"x-retries": retry_count},
146
+ ),
147
+ )
148
+
149
+ def _handle_retry(self, message: "EventType", ch: "BlockingChannel", method: "Basic.Deliver", retries: int) -> None:
150
+ ch.basic_ack(method.delivery_tag) # type: ignore[arg-type]
151
+ if retries < MAX_RETRIES:
152
+ delay = 2000 * (2**retries) # ms (exponential backoff)
153
+ delay_queue = f"{self.exchange}_retry_{delay}ms"
154
+
155
+ # Declare a delay queue with TTL
156
+ ch.queue_declare(
157
+ delay_queue,
158
+ durable=True,
159
+ arguments={
160
+ "x-dead-letter-exchange": self.exchange, # after delay → back to main
161
+ "x-message-ttl": delay,
162
+ },
163
+ )
164
+ ch.queue_bind(delay_queue, self.retry_exchange, routing_key="task")
165
+ self._basic_publish(message, retries + 1)
166
+ else:
167
+ logger.error(f"Dropping after {MAX_RETRIES} retries")
168
+
169
+ def listen(self, domains: list[str], callback: "PikaCallback", ack: bool = True) -> None:
170
+ if self.channel is None:
171
+ self.connect()
172
+ _callback = Callback(self, callback, ack=ack)
173
+ for domain in domains:
174
+ queue_name = f"{self.connection_name.lower()}_sub_to_{domain}"
175
+ self.channel.queue_declare(queue=queue_name, durable=True) # type: ignore[union-attr]
176
+ self.channel.queue_bind(exchange=self.exchange, queue=queue_name, routing_key=domain) # type: ignore[union-attr]
177
+ self.channel.basic_consume(queue=queue_name, on_message_callback=_callback, auto_ack=False) # type: ignore[union-attr]
178
+ self.channel.start_consuming() # type: ignore[union-attr]
179
+
180
+ def publish(self, message: "EventType", retry_count: int = 0, **kwargs: Any) -> None:
181
+ if not self.channel or self.channel.is_closed:
182
+ self.connect()
183
+ if self.channel:
184
+ logger.debug("publish to %s %s", self.exchange, message.get("domain", ""))
185
+ try:
186
+ self._basic_publish(message, 0)
187
+ except Exception as e: # noqa: BLE001
188
+ logger.critical("Unhandled error sending to RabbitMQ. Message not published.", exc_info=e)
189
+ else:
190
+ logger.critical("RabbitMQ connection not available after reconnect. Message not published.")
191
+
192
+ def close(self) -> None:
193
+ logger.debug("Closing RabbitMQ connection.")
194
+ try:
195
+ self.connection.close() # type: ignore[union-attr]
196
+ except (ConnectionClosedByBroker, AttributeError, ConnectionWrongStateError):
197
+ pass
198
+ finally:
199
+ self.connection = None
200
+ self.channel = None
@@ -13,7 +13,7 @@ DEFAULT_QUEUE_NAME = "django_model_changes"
13
13
 
14
14
  class StreamingConfig:
15
15
  _DEFAULTS = {
16
- "BROKER_URL": "redis://localhost:6379/0?queue=hope_changes",
16
+ "BROKER_URL": "console://",
17
17
  "RETRY_COUNT": 3,
18
18
  "RETRY_DELAY": 1,
19
19
  "MANAGER_CLASS": "streaming.manager.ChangeManager",
@@ -0,0 +1,23 @@
1
+ import logging
2
+
3
+ logger = logging.getLogger(__name__)
4
+
5
+
6
+ class StreamingError(Exception):
7
+ pass
8
+
9
+
10
+ class StreamingConfigError(StreamingError):
11
+ pass
12
+
13
+
14
+ class StreamingBackendError(StreamingError):
15
+ pass
16
+
17
+
18
+ class StreamingCallbackError(StreamingError):
19
+ """Generic exception raised when a callback fails. Message will be re-queued."""
20
+
21
+
22
+ class StreamingCallbackFailure(StreamingError): # noqa: N818
23
+ """Generic exception raised when a callback fails. Message will be discarded."""
@@ -1,3 +1,4 @@
1
+ import socket
1
2
  from typing import TYPE_CHECKING, Any
2
3
 
3
4
  if TYPE_CHECKING:
@@ -20,3 +21,10 @@ def make_event(message: "str | JSON", *, event: str = "", domain: str = "") -> "
20
21
  else:
21
22
  payload = message
22
23
  return {"event": event, "domain": domain, "payload": payload}
24
+
25
+
26
+ def get_local_ip() -> str:
27
+ s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
28
+ s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
29
+ s.connect(("<broadcast>", 12345)) # 12345 is random port. 0 fails on Mac.
30
+ return str(s.getsockname()[0])
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
28
28
  commit_id: COMMIT_ID
29
29
  __commit_id__: COMMIT_ID
30
30
 
31
- __version__ = version = '0.2'
32
- __version_tuple__ = version_tuple = (0, 2)
31
+ __version__ = version = '0.4'
32
+ __version_tuple__ = version_tuple = (0, 4)
33
33
 
34
34
  __commit_id__ = commit_id = None
@@ -1,126 +0,0 @@
1
- import atexit
2
- import json
3
- import logging
4
- import socket
5
- import time
6
- from typing import TYPE_CHECKING
7
-
8
- import pika.channel
9
- import pika.exceptions
10
- from pika import PlainCredentials
11
- from pika.exchange_type import ExchangeType
12
-
13
- from streaming.config import CONFIG
14
-
15
- from ..utils import DAY
16
- from ._base import BaseBackend
17
-
18
- if TYPE_CHECKING:
19
- from pika.adapters.blocking_connection import BlockingChannel
20
- from pika.spec import Basic, BasicProperties
21
-
22
- from streaming.types import EventType, PikaCallback
23
-
24
- logger = logging.getLogger(__name__)
25
-
26
-
27
- class RabbitMQBackend(BaseBackend):
28
- def __init__(self, url: str) -> None:
29
- super().__init__(url)
30
- self.host = str(self._parsed_url.hostname)
31
- self.port = int(self._parsed_url.port) if self._parsed_url.port else 5672
32
-
33
- self.exchange = self._options.get("exchange", "django-streaming-broadcast")
34
- self.connection_name = self._options.get("connection_name", "")
35
- self.timeout = float(self._options.get("timeout", 0.5))
36
- self.routing_key = self._options.get("routing_key", "")
37
- self.virtual_host = self._options.get("virtual_host", "/")
38
-
39
- self.connection: pika.BlockingConnection | None = None
40
- self.channel: BlockingChannel | None = None
41
-
42
- atexit.register(self.close)
43
-
44
- def connect(self) -> None:
45
- logger.debug("Connecting to %s:%s", self.host, self.port)
46
- if self.connection and self.connection.is_open:
47
- self.close()
48
-
49
- for __ in range(CONFIG.RETRY_COUNT):
50
- try:
51
- if self._parsed_url.username:
52
- auth = PlainCredentials(self._parsed_url.username, self._parsed_url.password or "")
53
- else:
54
- auth = PlainCredentials("guest", "guest")
55
- self.connection = pika.BlockingConnection(
56
- pika.ConnectionParameters(
57
- host=self.host,
58
- port=self.port,
59
- virtual_host=self.virtual_host,
60
- credentials=auth,
61
- socket_timeout=self.timeout,
62
- blocked_connection_timeout=self.timeout,
63
- stack_timeout=self.timeout,
64
- client_properties={
65
- "connection_name": self.connection_name,
66
- "product": "django-streaming",
67
- "information": "",
68
- "version": "1.0",
69
- },
70
- )
71
- )
72
- self.channel = self.connection.channel()
73
- self.channel.exchange_declare(exchange=self.exchange, exchange_type=ExchangeType.topic, durable=True)
74
- self.channel.queue_declare(queue="global_queue", durable=True)
75
- self.channel.queue_bind(exchange=self.exchange, queue="global_queue", routing_key="#")
76
- return
77
- except (socket.gaierror, pika.exceptions.AMQPConnectionError):
78
- logger.warning(
79
- f"Could not connect to RabbitMQ. Retrying in {CONFIG.RETRY_DELAY} seconds...",
80
- )
81
- time.sleep(CONFIG.RETRY_DELAY)
82
- logger.critical("Could not connect to RabbitMQ after multiple retries.")
83
-
84
- def listen(self, domains: list[str], callback: "PikaCallback") -> None:
85
- def _callback(
86
- ch: "BlockingChannel", method: "Basic.Deliver", properties: "BasicProperties", body: bytes
87
- ) -> None:
88
- callback(ch, method, properties, body)
89
- ch.basic_ack(delivery_tag=method.delivery_tag) # type: ignore[arg-type]
90
-
91
- if self.channel is None:
92
- self.connect()
93
-
94
- for domain in domains:
95
- queue_name = f"{self.connection_name.lower()}_sub_to_{domain}"
96
- self.channel.queue_declare(queue=queue_name, durable=True) # type: ignore[union-attr]
97
- self.channel.queue_bind(exchange=self.exchange, queue=queue_name, routing_key=domain) # type: ignore[union-attr]
98
- self.channel.basic_consume(queue=queue_name, on_message_callback=_callback, auto_ack=False) # type: ignore[union-attr]
99
- self.channel.start_consuming() # type: ignore[union-attr]
100
-
101
- def publish(self, message: "EventType") -> None:
102
- if not self.channel or self.channel.is_closed:
103
- self.connect()
104
- if self.channel:
105
- logger.debug("publish to %s %s", self.exchange, message.get("domain", ""))
106
- try:
107
- self.channel.basic_publish(
108
- exchange=self.exchange,
109
- routing_key=message.get("domain", self.routing_key) or self.routing_key,
110
- body=json.dumps(message).encode(),
111
- properties=pika.BasicProperties(
112
- delivery_mode=pika.spec.PERSISTENT_DELIVERY_MODE,
113
- expiration=str(DAY * 2 * 1000), # milliseconds
114
- ),
115
- )
116
- except Exception as e: # noqa: BLE001
117
- logger.critical("Unhandled error sending to RabbitMQ. Message not published.", exc_info=e)
118
- else:
119
- logger.critical("RabbitMQ connection not available after reconnect. Message not published.")
120
-
121
- def close(self) -> None:
122
- if self.connection and self.connection.is_open:
123
- logger.debug("Closing RabbitMQ connection.")
124
- self.connection.close()
125
- self.connection = None
126
- self.channel = None
@@ -1,56 +0,0 @@
1
- import json
2
- import logging
3
- import time
4
- from typing import TYPE_CHECKING
5
-
6
- import redis
7
- from redis.exceptions import ConnectionError as RedisConnectionError
8
-
9
- from streaming.config import CONFIG
10
- from streaming.exceptions import StreamingBackendError
11
-
12
- from ._base import BaseBackend
13
-
14
- if TYPE_CHECKING:
15
- from redis import Redis
16
-
17
- from ..types import EventType
18
-
19
- logger = logging.getLogger(__name__)
20
-
21
-
22
- class RedisBackend(BaseBackend):
23
- redis_client: "Redis[bytes]"
24
-
25
- def __init__(self, url: str) -> None:
26
- super().__init__(url)
27
- self.db = int(self._parsed_url.path.lstrip("/") or 0)
28
- self.host = str(self._parsed_url.hostname)
29
- self.port = int(self._parsed_url.port) if self._parsed_url.port else 6379
30
- self.timeout = float(self._options.get("timeout", 0.5))
31
-
32
- self.redis_client = self._get_client()
33
-
34
- def _get_client(self) -> "Redis[bytes]":
35
- for __ in range(CONFIG.RETRY_COUNT):
36
- try:
37
- client = redis.Redis(
38
- host=self.host,
39
- port=self.port,
40
- db=self.db,
41
- socket_timeout=self.timeout,
42
- socket_connect_timeout=self.timeout,
43
- )
44
- client.ping()
45
- return client
46
- except RedisConnectionError:
47
- logger.warning("Could not connect to Redis. Retrying in %s seconds...", CONFIG.RETRY_DELAY)
48
- time.sleep(CONFIG.RETRY_DELAY)
49
- raise StreamingBackendError("Could not connect to Redis after multiple retries.")
50
-
51
- def publish(self, message: "EventType") -> None:
52
- try:
53
- self.redis_client.publish(self.queue_name, json.dumps(message).encode())
54
- except RedisConnectionError:
55
- self.redis_client = self._get_client()
56
- self.redis_client.publish(self.queue_name, json.dumps(message).encode())
@@ -1,15 +0,0 @@
1
- import logging
2
-
3
- logger = logging.getLogger(__name__)
4
-
5
-
6
- class StreamingError(Exception):
7
- pass
8
-
9
-
10
- class StreamingConfigError(StreamingError):
11
- pass
12
-
13
-
14
- class StreamingBackendError(StreamingError):
15
- pass
File without changes