django-streaming 0.7.2__tar.gz → 0.9.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. {django_streaming-0.7.2 → django_streaming-0.9.0}/PKG-INFO +5 -6
  2. {django_streaming-0.7.2 → django_streaming-0.9.0}/pyproject.toml +23 -7
  3. {django_streaming-0.7.2 → django_streaming-0.9.0}/src/streaming/__cli__.py +6 -6
  4. {django_streaming-0.7.2 → django_streaming-0.9.0}/src/streaming/backends/__init__.py +4 -4
  5. {django_streaming-0.7.2 → django_streaming-0.9.0}/src/streaming/backends/rabbitmq.py +75 -17
  6. {django_streaming-0.7.2 → django_streaming-0.9.0}/src/streaming/config.py +1 -0
  7. {django_streaming-0.7.2 → django_streaming-0.9.0}/src/streaming/manager.py +1 -1
  8. {django_streaming-0.7.2 → django_streaming-0.9.0}/src/streaming/utils.py +1 -1
  9. django_streaming-0.9.0/src/streaming/version.py +24 -0
  10. django_streaming-0.7.2/src/streaming/version.py +0 -34
  11. {django_streaming-0.7.2 → django_streaming-0.9.0}/.gitignore +0 -0
  12. {django_streaming-0.7.2 → django_streaming-0.9.0}/LICENSE +0 -0
  13. {django_streaming-0.7.2 → django_streaming-0.9.0}/README.md +0 -0
  14. {django_streaming-0.7.2 → django_streaming-0.9.0}/src/streaming/__init__.py +0 -0
  15. {django_streaming-0.7.2 → django_streaming-0.9.0}/src/streaming/apps.py +0 -0
  16. {django_streaming-0.7.2 → django_streaming-0.9.0}/src/streaming/backends/_base.py +0 -0
  17. {django_streaming-0.7.2 → django_streaming-0.9.0}/src/streaming/backends/console.py +0 -0
  18. {django_streaming-0.7.2 → django_streaming-0.9.0}/src/streaming/backends/debug.py +0 -0
  19. {django_streaming-0.7.2 → django_streaming-0.9.0}/src/streaming/callbacks.py +0 -0
  20. {django_streaming-0.7.2 → django_streaming-0.9.0}/src/streaming/event.py +0 -0
  21. {django_streaming-0.7.2 → django_streaming-0.9.0}/src/streaming/exceptions.py +0 -0
  22. {django_streaming-0.7.2 → django_streaming-0.9.0}/src/streaming/py.typed +0 -0
  23. {django_streaming-0.7.2 → django_streaming-0.9.0}/src/streaming/types.py +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: django-streaming
3
- Version: 0.7.2
3
+ Version: 0.9.0
4
4
  Summary: A project for streaming data.
5
5
  Project-URL: Documentation, https://github.com/UNICEF/django-streaming
6
6
  Project-URL: Homepage, https://github.com/UNICEF/django-streaming
@@ -8,27 +8,26 @@ Author-email: Stefano Apostolico <s.apostolico@gmail.com>
8
8
  License-File: LICENSE
9
9
  Classifier: Environment :: Web Environment
10
10
  Classifier: Framework :: Django
11
- Classifier: Framework :: Django :: 3.2
12
11
  Classifier: Framework :: Django :: 4.2
13
12
  Classifier: Framework :: Django :: 5.2
13
+ Classifier: Framework :: Django :: 6.0
14
14
  Classifier: Intended Audience :: Developers
15
15
  Classifier: License :: OSI Approved :: MIT License
16
16
  Classifier: Operating System :: OS Independent
17
17
  Classifier: Programming Language :: Python :: 3 :: Only
18
- Classifier: Programming Language :: Python :: 3.10
19
18
  Classifier: Programming Language :: Python :: 3.11
20
19
  Classifier: Programming Language :: Python :: 3.12
21
20
  Classifier: Programming Language :: Python :: 3.13
22
21
  Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
23
22
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
24
- Requires-Python: >=3.10
23
+ Requires-Python: >=3.11
25
24
  Requires-Dist: click>=8
26
25
  Requires-Dist: colorama>=0.4.6
27
26
  Requires-Dist: django-stubs-ext
28
27
  Requires-Dist: django>=3
29
28
  Requires-Dist: markdown>=3.8.2
30
29
  Requires-Dist: pika>=1.3.2
31
- Requires-Dist: requests>=2.32.5
30
+ Requires-Dist: requests>=2.33
32
31
  Description-Content-Type: text/markdown
33
32
 
34
33
  # Django Streaming
@@ -12,18 +12,17 @@ readme = "README.md"
12
12
  authors = [
13
13
  { name = "Stefano Apostolico", email = "s.apostolico@gmail.com" },
14
14
  ]
15
- requires-python = ">=3.10"
15
+ requires-python = ">=3.11"
16
16
  classifiers = [
17
17
  "Environment :: Web Environment",
18
18
  "Framework :: Django",
19
- "Framework :: Django :: 3.2",
20
19
  "Framework :: Django :: 4.2",
21
20
  "Framework :: Django :: 5.2",
21
+ "Framework :: Django :: 6.0",
22
22
  "Intended Audience :: Developers",
23
23
  "License :: OSI Approved :: MIT License",
24
24
  "Operating System :: OS Independent",
25
25
  "Programming Language :: Python :: 3 :: Only",
26
- "Programming Language :: Python :: 3.10",
27
26
  "Programming Language :: Python :: 3.11",
28
27
  "Programming Language :: Python :: 3.12",
29
28
  "Programming Language :: Python :: 3.13",
@@ -40,7 +39,7 @@ dependencies = [
40
39
  "django-stubs-ext",
41
40
  "markdown>=3.8.2",
42
41
  "pika>=1.3.2",
43
- "requests>=2.32.5",
42
+ "requests>=2.33",
44
43
  ]
45
44
 
46
45
  urls.Documentation = "https://github.com/UNICEF/django-streaming"
@@ -54,7 +53,7 @@ dev = [
54
53
  "factory-boy>=3.3.3",
55
54
  "pre-commit>=3.3.3",
56
55
  "pyrabbit2>=1.0.7",
57
- "pytest>=8.4.1",
56
+ "pytest>=9.0.3",
58
57
  "pytest-cov>=6.2.1",
59
58
  "pytest-django>=4.11.1",
60
59
  "pytest-responses>=0.5.1",
@@ -79,12 +78,12 @@ docs = [
79
78
  "mkdocs-include-markdown-plugin>=6.2.2",
80
79
  "mkdocs-link-marker>=0.1.3",
81
80
  "mkdocs-macros-plugin>=1.0.5",
82
- "mkdocs-material>=9.5.15",
81
+ "mkdocs-material>=9.7.7",
83
82
  "mkdocs-minify-plugin>=0.8",
84
83
  "mkdocs-redirects>=1.2.1",
85
84
  "mkdocs-simple-hooks>=0.1.5",
86
85
  "mkdocstrings[python]>=0.27",
87
- "pymdown-extensions>=10.14",
86
+ "pymdown-extensions>=11.0.1",
88
87
  ]
89
88
  type-check = [
90
89
  "django-stubs[compatible-mypy]",
@@ -107,3 +106,20 @@ include = [
107
106
 
108
107
  [tool.hatch.build.targets.wheel]
109
108
  packages = [ "src/streaming" ]
109
+
110
+ [tool.uv]
111
+ constraint-dependencies = [
112
+ "certifi>=2026.10.18",
113
+ "filelock>=4.2.1",
114
+ "idna>=3.15",
115
+ "pillow>=13",
116
+ "pip>=26.2",
117
+ "pygments>=2.20",
118
+ "setuptools>=84.1",
119
+ "soupsieve>=2.9",
120
+ "sqlparse>=0.6",
121
+ "urllib3>=2.7",
122
+ "uv>=0.13.4",
123
+ "virtualenv>=21.14.9",
124
+ "webob>=1.8.11",
125
+ ]
@@ -56,7 +56,7 @@ def configure_logging(debug: bool, loggers: Iterable[str] = ("streaming",)) -> N
56
56
  def cli() -> None:
57
57
  """Streaming CLI."""
58
58
  try:
59
- import django
59
+ import django # noqa
60
60
 
61
61
  django.setup()
62
62
  except ModuleNotFoundError as e:
@@ -69,7 +69,7 @@ def cli() -> None:
69
69
  @click.option("--queues/--no-queues", "queues", is_flag=True, default=False, help="Debug mode")
70
70
  @click.option("--debug", is_flag=True, help="Debug mode")
71
71
  def configure(queues: bool = False, debug: bool = False) -> None:
72
- from streaming.config import CONFIG
72
+ from streaming.config import CONFIG # noqa
73
73
 
74
74
  routing = None
75
75
  configure_logging(debug)
@@ -161,9 +161,9 @@ def listen( # noqa PLR0913
161
161
  @cli.command()
162
162
  def purge() -> None:
163
163
  """Purges all messages from the configured queues."""
164
- from streaming.backends.rabbitmq import RabbitMQBackend
165
- from streaming.config import CONFIG
166
- from streaming.manager import initialize_engine
164
+ from streaming.backends.rabbitmq import RabbitMQBackend # noqa
165
+ from streaming.config import CONFIG # noqa
166
+ from streaming.manager import initialize_engine # noqa
167
167
 
168
168
  manager = initialize_engine(True)
169
169
  backend = manager.backend
@@ -184,7 +184,7 @@ def purge() -> None:
184
184
  @cli.command()
185
185
  def check() -> None:
186
186
  """Checks the streaming configuration and connection."""
187
- from streaming.config import CONFIG
187
+ from streaming.config import CONFIG # noqa
188
188
 
189
189
  click.secho("System Configuration:")
190
190
  config_dict = dict(CONFIG._parsed)
@@ -14,7 +14,7 @@ __all__ = ["RabbitMQBackend", "get_backend"]
14
14
 
15
15
 
16
16
  def get_backend() -> "BaseBackend":
17
- from streaming.config import CONFIG
17
+ from streaming.config import CONFIG # noqa
18
18
 
19
19
  if not CONFIG.BROKER_URL:
20
20
  raise StreamingConfigError("Empty BROKER_URL")
@@ -22,15 +22,15 @@ def get_backend() -> "BaseBackend":
22
22
  parsed_url = urlparse(CONFIG.BROKER_URL)
23
23
 
24
24
  if parsed_url.scheme == "console":
25
- from .console import ConsoleBackend
25
+ from .console import ConsoleBackend # noqa
26
26
 
27
27
  return ConsoleBackend(CONFIG.BROKER_URL)
28
28
  if parsed_url.scheme == "rabbit":
29
- from .rabbitmq import RabbitMQBackend
29
+ from .rabbitmq import RabbitMQBackend # noqa
30
30
 
31
31
  return RabbitMQBackend(CONFIG.BROKER_URL)
32
32
  if parsed_url.scheme == "debug":
33
- from .debug import DebugBackend
33
+ from .debug import DebugBackend # noqa
34
34
 
35
35
  return DebugBackend(CONFIG.BROKER_URL)
36
36
  raise StreamingConfigError(f"Broker not supported: '{parsed_url.scheme}://{parsed_url.hostname}:{parsed_url.port}'")
@@ -3,6 +3,7 @@ import socket
3
3
  import time
4
4
  from collections import defaultdict
5
5
  from contextlib import suppress
6
+ from copy import copy
6
7
  from typing import TYPE_CHECKING
7
8
 
8
9
  import pika.channel
@@ -32,8 +33,6 @@ if TYPE_CHECKING:
32
33
 
33
34
  logger = logging.getLogger(__name__)
34
35
 
35
- MAX_RETRIES = 5
36
-
37
36
 
38
37
  class Callback:
39
38
  def __init__(self, queue_name: str, backend: "RabbitMQBackend", user_callback: "UserCallback") -> None:
@@ -50,14 +49,51 @@ class Callback:
50
49
  except CallbackSkipAck as e:
51
50
  logger.error(f"StreamingCallbackSkipAckException: {e}", exc_info=e)
52
51
  except CallbackRetry:
53
- evt: Event = Event.unmarshal(body)
54
52
  retries = int(properties.headers.get("x-retries", 0)) # type: ignore[union-attr]
55
- self.backend._handle_retry(evt, ch, method, retries)
53
+ if retries >= CONFIG.CALLBACK_RETRY_COUNT:
54
+ self._handle_exhausted(ch, method, properties, body)
55
+ else:
56
+ self.backend._handle_retry(Event.unmarshal(body), ch, method, retries)
56
57
  except CallbackError as e:
57
58
  logger.error(f"Callback failure: {e}", exc_info=e)
58
59
  except Exception as e:
59
60
  logger.exception(f"Unexpected exception occurred: {e}", exc_info=e)
60
61
 
62
+ def _handle_exhausted(
63
+ self, ch: "BlockingChannel", method: "Basic.Deliver", properties: "BasicProperties", body: bytes
64
+ ) -> None:
65
+ """Save an exhausted message before acknowledging the original delivery."""
66
+ message_id = Event.unmarshal(body).id
67
+ queue = f"{self.backend.get_real_queue_name(self.queue_name)}_exhausted"
68
+ exchange = self.backend.retry_exchange
69
+ try:
70
+ ch.queue_declare(queue=queue, durable=True)
71
+ ch.queue_bind(queue=queue, exchange=exchange, routing_key=queue)
72
+
73
+ headers = dict(properties.headers or {})
74
+ for key in ("CC", "BCC"):
75
+ if key in headers:
76
+ headers[f"x-original-{key}"] = headers.pop(key)
77
+ headers.update({"x-original-exchange": method.exchange, "x-original-routing-key": method.routing_key})
78
+ if properties.user_id is not None:
79
+ headers["x-original-user-id"] = properties.user_id
80
+
81
+ saved = copy(properties)
82
+ saved.headers = headers
83
+ saved.delivery_mode = pika.spec.PERSISTENT_DELIVERY_MODE
84
+ saved.expiration = None
85
+ saved.user_id = None
86
+ ch.basic_publish(exchange=exchange, routing_key=queue, body=body, properties=saved, mandatory=True)
87
+ except Exception:
88
+ logger.exception("Failed to save exhausted message %s to %s", message_id, queue)
89
+ if not ch.is_open:
90
+ raise
91
+ ch.basic_nack(method.delivery_tag, requeue=True) # type: ignore[arg-type]
92
+ return
93
+
94
+ ch.basic_ack(method.delivery_tag) # type: ignore[arg-type]
95
+ logger.error("Saved exhausted message %s to %s", message_id, queue)
96
+
61
97
 
62
98
  class RabbitMQBackend(BaseBackend):
63
99
  def __init__(self, url: str) -> None:
@@ -175,12 +211,18 @@ class RabbitMQBackend(BaseBackend):
175
211
  # Listener
176
212
 
177
213
  def _basic_publish(
178
- self, message: "Event", routing_key: str, retry_count: int = 0, exchange: str | None = None
214
+ self,
215
+ message: "Event",
216
+ routing_key: str,
217
+ retry_count: int = 0,
218
+ exchange: str | None = None,
219
+ mandatory: bool = False,
179
220
  ) -> None:
180
221
  self.check_channel()
181
222
  self.channel.basic_publish( # type: ignore[union-attr]
182
223
  exchange=exchange or self.exchange,
183
224
  routing_key=routing_key,
225
+ mandatory=mandatory,
184
226
  body=message.marshall(),
185
227
  properties=pika.BasicProperties(
186
228
  delivery_mode=pika.spec.PERSISTENT_DELIVERY_MODE,
@@ -190,15 +232,14 @@ class RabbitMQBackend(BaseBackend):
190
232
  )
191
233
 
192
234
  def _handle_retry(self, message: "Event", ch: "BlockingChannel", method: "Basic.Deliver", retries: int) -> None:
193
- ch.basic_ack(method.delivery_tag) # type: ignore[arg-type]
194
- if retries < MAX_RETRIES:
195
- delay = 2000 * (2**retries) # ms (exponential backoff)
196
- delay_queue = f"{self.exchange}_retry_{method.routing_key}_{delay}ms"
197
- logger.debug(
198
- f"Retrying message {message.id} ({retries + 1}/{MAX_RETRIES}) "
199
- f"with routing key '{method.routing_key}' in {delay / 1000}s"
200
- )
201
-
235
+ """Publish a delayed retry before acknowledging the original message."""
236
+ delay = 2000 * (2**retries) # ms (exponential backoff)
237
+ delay_queue = f"{self.exchange}_retry_{method.routing_key}_{delay}ms"
238
+ logger.debug(
239
+ f"Retrying message {message.id} ({retries + 1}/{CONFIG.CALLBACK_RETRY_COUNT}) "
240
+ f"with routing key '{method.routing_key}' in {delay / 1000}s"
241
+ )
242
+ try:
202
243
  # Declare a delay queue with TTL
203
244
  ch.queue_declare(
204
245
  delay_queue,
@@ -210,14 +251,31 @@ class RabbitMQBackend(BaseBackend):
210
251
  },
211
252
  )
212
253
  ch.queue_bind(delay_queue, self.retry_exchange, routing_key=delay_queue)
213
- self._basic_publish(message, routing_key=delay_queue, retry_count=retries + 1, exchange=self.retry_exchange)
214
- else:
215
- logger.error(f"Dropping message {message.id} after {MAX_RETRIES} retries")
254
+ self._basic_publish(
255
+ message,
256
+ routing_key=delay_queue,
257
+ retry_count=retries + 1,
258
+ exchange=self.retry_exchange,
259
+ mandatory=True,
260
+ )
261
+ except Exception as e:
262
+ logger.exception(f"Failed to schedule retry for message {message.id}; leaving it unacknowledged: {e}")
263
+ if ch.is_open:
264
+ # Channel still usable: return the message to the queue instead of leaving it stuck
265
+ ch.basic_nack(method.delivery_tag, requeue=True) # type: ignore[arg-type]
266
+ else:
267
+ # Channel closed: broker redelivers unacked messages automatically
268
+ raise
269
+ return
270
+ ch.basic_ack(method.delivery_tag) # type: ignore[arg-type]
216
271
 
217
272
  def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
218
273
  if self.channel is None:
219
274
  self.connect()
220
275
  self.check_channel()
276
+ # Publisher confirms are enabled only for the listener's channel (needed so retry publications are
277
+ # confirmed before the original message is acked); regular publish() is not slowed down by them.
278
+ self.channel.confirm_delivery() # type: ignore[union-attr]
221
279
  configured_queues = CONFIG.QUEUES
222
280
  if not configured_queues:
223
281
  logger.warning("No queues configured in settings.STREAMING['QUEUES']")
@@ -19,6 +19,7 @@ class StreamingConfig:
19
19
  "MESSAGE_TTL": 60 * 60 * 24 * 2, # 2 days
20
20
  "MANAGER_CLASS": "streaming.manager.ThreadedChangeManager",
21
21
  "LISTEN_CALLBACK": "streaming.callbacks.default_callback",
22
+ "CALLBACK_RETRY_COUNT": 5,
22
23
  }
23
24
 
24
25
  def __init__(self) -> None:
@@ -99,7 +99,7 @@ class ThreadedChangeManager(ChangeManager):
99
99
 
100
100
 
101
101
  def get_manager() -> ChangeManager:
102
- from streaming.config import CONFIG
102
+ from streaming.config import CONFIG # noqa
103
103
 
104
104
  try:
105
105
  return import_string(CONFIG.MANAGER_CLASS)() # type: ignore[no-any-return]
@@ -52,7 +52,7 @@ def parse_bool(value: Any) -> bool:
52
52
 
53
53
 
54
54
  def make_event(message: "str|JSON", *, value_type: "EventType" = "absolute", message_id: str | None = None) -> "Event":
55
- from streaming.event import Event
55
+ from streaming.event import Event # noqa
56
56
 
57
57
  return Event.build(
58
58
  data=message,
@@ -0,0 +1,24 @@
1
+ # file generated by vcs-versioning
2
+ # don't change, don't track in version control
3
+ from __future__ import annotations
4
+
5
+ __all__ = [
6
+ "__version__",
7
+ "__version_tuple__",
8
+ "version",
9
+ "version_tuple",
10
+ "__commit_id__",
11
+ "commit_id",
12
+ ]
13
+
14
+ version: str
15
+ __version__: str
16
+ __version_tuple__: tuple[int | str, ...]
17
+ version_tuple: tuple[int | str, ...]
18
+ commit_id: str | None
19
+ __commit_id__: str | None
20
+
21
+ __version__ = version = '0.9.0'
22
+ __version_tuple__ = version_tuple = (0, 9, 0)
23
+
24
+ __commit_id__ = commit_id = None
@@ -1,34 +0,0 @@
1
- # file generated by setuptools-scm
2
- # don't change, don't track in version control
3
-
4
- __all__ = [
5
- "__version__",
6
- "__version_tuple__",
7
- "version",
8
- "version_tuple",
9
- "__commit_id__",
10
- "commit_id",
11
- ]
12
-
13
- TYPE_CHECKING = False
14
- if TYPE_CHECKING:
15
- from typing import Tuple
16
- from typing import Union
17
-
18
- VERSION_TUPLE = Tuple[Union[int, str], ...]
19
- COMMIT_ID = Union[str, None]
20
- else:
21
- VERSION_TUPLE = object
22
- COMMIT_ID = object
23
-
24
- version: str
25
- __version__: str
26
- __version_tuple__: VERSION_TUPLE
27
- version_tuple: VERSION_TUPLE
28
- commit_id: COMMIT_ID
29
- __commit_id__: COMMIT_ID
30
-
31
- __version__ = version = '0.7.2'
32
- __version_tuple__ = version_tuple = (0, 7, 2)
33
-
34
- __commit_id__ = commit_id = None