django-streaming 0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,52 @@
1
+ Metadata-Version: 2.4
2
+ Name: django-streaming
3
+ Version: 0.1
4
+ Summary: A project for streaming data.
5
+ Project-URL: Documentation, https://github.com/UNICEF/django-streaming
6
+ Project-URL: Homepage, https://github.com/UNICEF/django-streaming
7
+ Author-email: Stefano Apostolico <s.apostolico@gmail.com>
8
+ License-File: LICENSE
9
+ Classifier: Environment :: Web Environment
10
+ Classifier: Framework :: Django
11
+ Classifier: Framework :: Django :: 4.2
12
+ Classifier: Framework :: Django :: 5.2
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Programming Language :: Python :: 3 :: Only
17
+ Classifier: Programming Language :: Python :: 3.9
18
+ Classifier: Programming Language :: Python :: 3.10
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Programming Language :: Python :: 3.13
22
+ Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
23
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
24
+ Requires-Python: >=3.9
25
+ Requires-Dist: click>=8
26
+ Requires-Dist: django-stubs-ext
27
+ Requires-Dist: django>=4
28
+ Requires-Dist: markdown>=3.8.2
29
+ Requires-Dist: requests>=2.32.5
30
+ Provides-Extra: rabbitmq
31
+ Requires-Dist: pika>=1.3.2; extra == 'rabbitmq'
32
+ Provides-Extra: redis
33
+ Requires-Dist: redis>=5; extra == 'redis'
34
+ Description-Content-Type: text/markdown
35
+
36
+ # Django Streaming
37
+
38
+ [![Test](https://github.com/unicef/django-streaming/actions/workflows/test.yml/badge.svg)](https://github.com/unicef/django-streaming/actions/workflows/test.yml)
39
+ [![Lint](https://github.com/unicef/django-streaming/actions/workflows/lint.yml/badge.svg)](https://github.com/unicef/django-streaming/actions/workflows/lint.yml)
40
+ [![codecov](https://codecov.io/github/unicef/django-streaming/branch/develop/graph/badge.svg?token=3ZmxTFfYra)](https://codecov.io/github/unicef/django-streaming)
41
+ [![Documentation](https://github.com/unicef/django-streaming/actions/workflows/docs.yml/badge.svg)](https://unicef.github.io/django-streaming/)
42
+ [![Pypi](https://badge.fury.io/py/django-streaming.svg)](https://badge.fury.io/py/django-streaming)
43
+
44
+
45
+ `django-streaming` is a robust and flexible Django application designed for efficiently streaming data from your
46
+ Django projects to various message brokers.
47
+
48
+ It provides a pluggable backend architecture supporting popular options like RabbitMQ and Redis, alongside console and
49
+ debug backends for development. With features like a threaded change manager for non-blocking asynchronous
50
+ notifications, configurable connection retry mechanisms, and graceful shutdown handling, django-streaming ensures
51
+ reliable data delivery. It seamlessly integrates with Django's ORM via signals to automatically stream model changes
52
+ and offers a command-line interface for easy interaction and debugging."
@@ -0,0 +1,22 @@
1
+ streaming/__cli__.py,sha256=EQ07hTrP6HEYo9Z1SEPvJqg6oHm1AdDQi3i0GzUn3cI,4071
2
+ streaming/api.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
3
+ streaming/apps.py,sha256=H_aGrsRFMZ4YxZsFUxEGf07OdtpkRyvRgJMKzXXEvm4,373
4
+ streaming/config.py,sha256=yYTvmzJcGqbKi996dx3oXY65If9QnDH8matap8a-KaE,1227
5
+ streaming/exceptions.py,sha256=GaiNr__q32JrZYmAlHj6JtUakJM3J-7NutXpYrrak_4,208
6
+ streaming/handlers.py,sha256=vU7QiGSy_AJuJFmClUocqIJ-Ltku_8C24ZU8L6fLJR0,53
7
+ streaming/manager.py,sha256=FxLbECrWrmGEVqvd9kZqmv7k172x0UAekhV2LNQRp1E,2962
8
+ streaming/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
9
+ streaming/types.pyi,sha256=IPRxv-oqA59MPpHIuBse8In36r9FxUfuIgM0N--rx6Q,443
10
+ streaming/utils.py,sha256=xw_LpLpQXVcfvvFrhNzlZBKNy6taYPuCoKqbubDWq84,589
11
+ streaming/version.py,sha256=zCgmOZqbKglNFIB9lB_sgjMOR6jrNzUWcCp-9rpWq18,699
12
+ streaming/backends/__init__.py,sha256=Mi2t4WxT0z1l0gKb6cQwqzbEXcs-9GgP9RJVW6CDWdw,948
13
+ streaming/backends/_base.py,sha256=YSW9B8oYSalivZdagwGZeKJlojUK3652Q3uZonaLmVg,753
14
+ streaming/backends/console.py,sha256=MpM8xcegbNon11x6wMHu1cJZQSkJFFLSqtXa1LFFtP0,486
15
+ streaming/backends/debug.py,sha256=dxaBtWmZpypxQrMfNkdzjy6ynuFxV6KunHZZyBN2faw,414
16
+ streaming/backends/rabbitmq.py,sha256=I3RmisjiM0czGems6mz96FUj_AjwJgtnVDK50hPys-M,5190
17
+ streaming/backends/redis.py,sha256=3LaTN5MQe1FhOLk_x7ihCYo5ZIHby5MNUjpfr2pLzEU,1892
18
+ django_streaming-0.1.dist-info/METADATA,sha256=kusigekiczFiRxIVQArSL9IjZD8rqql_392wDnQnXBk,2859
19
+ django_streaming-0.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
20
+ django_streaming-0.1.dist-info/entry_points.txt,sha256=qaKNtkQAveXwcCAg-WNnsggvLdk2hsgnI3e3diebA6s,49
21
+ django_streaming-0.1.dist-info/licenses/LICENSE,sha256=Naw8hAJurUrBlkD0Q6kG8MvljMnQT0Rop7WPS8VOFOc,1063
22
+ django_streaming-0.1.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.27.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ stream = streaming.__cli__:cli
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 UNICEF
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
streaming/__cli__.py ADDED
@@ -0,0 +1,159 @@
1
+ import datetime
2
+ import json
3
+ import logging
4
+ import random
5
+ from typing import TYPE_CHECKING
6
+
7
+ import click
8
+ from pika.adapters.blocking_connection import BlockingChannel
9
+ from pika.spec import Basic, BasicProperties
10
+
11
+ from streaming.utils import make_event
12
+
13
+ if TYPE_CHECKING:
14
+ from streaming.types import EventType
15
+
16
+ logger = logging.getLogger(__name__)
17
+
18
+ names = [
19
+ "Spider-Man",
20
+ "Peter Parker",
21
+ "Iron Man",
22
+ "Tony Stark",
23
+ "Captain America",
24
+ "Steve Rogers",
25
+ "Thor",
26
+ "Thor Odinson",
27
+ "Hulk",
28
+ "Bruce Banner",
29
+ "Black Widow",
30
+ "Natasha Romanoff",
31
+ "Hawkeye",
32
+ "Clint Barton",
33
+ "Doctor Strange",
34
+ "Stephen Strange",
35
+ "Black Panther",
36
+ "T’Challa",
37
+ "Scarlet Witch",
38
+ "Wanda Maximoff",
39
+ "Vision",
40
+ "Vision",
41
+ "Ant-Man",
42
+ "Scott Lang",
43
+ "Wasp",
44
+ "Hope van Dyne",
45
+ "Falcon",
46
+ "Sam Wilson",
47
+ "Winter Soldier",
48
+ "Bucky Barnes",
49
+ "Captain Marvel",
50
+ "Carol Danvers",
51
+ "Mr. Fantastic",
52
+ "Reed Richards",
53
+ "Invisible Woman",
54
+ "Sue Storm",
55
+ "Human Torch",
56
+ "Johnny Storm",
57
+ "The Thing",
58
+ "Ben Grimm",
59
+ "Wolverine",
60
+ "James Howlett",
61
+ "Cyclops",
62
+ "Scott Summers",
63
+ "Jean Grey",
64
+ "Jean Grey",
65
+ "Storm",
66
+ "Ororo Munroe",
67
+ "Professor X",
68
+ "Charles Xavier",
69
+ "Rogue",
70
+ "Anna Marie",
71
+ "Gambit",
72
+ "Remy LeBeau",
73
+ "Beast",
74
+ "Hank McCoy",
75
+ "Colossus",
76
+ "Piotr Rasputin",
77
+ "Nightcrawler",
78
+ "Kurt Wagner",
79
+ ]
80
+
81
+
82
+ @click.group()
83
+ def cli() -> None:
84
+ """Streaming CLI."""
85
+ import django
86
+
87
+ django.setup()
88
+
89
+
90
+ @cli.group()
91
+ def rabbit() -> None:
92
+ """RabbitMQ client."""
93
+
94
+
95
+ @rabbit.command()
96
+ @click.option("--message", default="Test Message", help="Message to send")
97
+ @click.option("--domain", default="", help="Consumer name")
98
+ def send(message: str, domain: str) -> None:
99
+ from streaming.backends.rabbitmq import RabbitMQBackend
100
+ from streaming.manager import initialize_engine
101
+
102
+ manager = initialize_engine(True)
103
+
104
+ backend = manager.backend
105
+ if not isinstance(backend, RabbitMQBackend):
106
+ raise click.ClickException("RabbitMQ backend is not configured. Please set BROKER_URL to a rabbit:// URL.")
107
+
108
+ backend.connection_name = "sender"
109
+ backend.connect()
110
+ timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
111
+
112
+ try:
113
+ payload = json.loads(message)
114
+ except json.decoder.JSONDecodeError:
115
+ payload = {
116
+ "timestamp": timestamp,
117
+ "message": message,
118
+ }
119
+ msg: EventType = make_event(payload, event="Test", domain=domain)
120
+ backend.publish(msg)
121
+ click.secho(f"Server: {backend.host}:{backend.port}")
122
+ click.secho(f"Publish to: {backend.exchange} {domain}")
123
+ click.secho(f"Sent: {msg}")
124
+ backend.connection.close() # type: ignore[union-attr]
125
+
126
+
127
+ @rabbit.command()
128
+ @click.option("--name", default=None, help="Consumer name")
129
+ @click.option("--domain", default="", help="Domain name")
130
+ def listen(name: str, domain: str) -> None:
131
+ from streaming.backends.rabbitmq import RabbitMQBackend
132
+ from streaming.manager import initialize_engine
133
+
134
+ manager = initialize_engine(True)
135
+ backend = manager.backend
136
+
137
+ if not isinstance(backend, RabbitMQBackend):
138
+ raise click.ClickException("RabbitMQ backend is not configured. Please set BROKER_URL to a rabbit:// URL.")
139
+
140
+ if not name:
141
+ name = random.choice(names) # noqa S311
142
+ if name != backend.connection_name:
143
+ backend.connection_name = name
144
+ backend.connect()
145
+
146
+ click.secho(f"Server: {backend.host}:{backend.port}")
147
+ click.secho(f"Consumer: {name}")
148
+ click.secho(f"Listen on: {backend.exchange} {domain}")
149
+
150
+ def callback(ch: BlockingChannel, method: Basic.Deliver, properties: BasicProperties, body: bytes) -> None:
151
+ click.echo(f"Received {body.decode()}")
152
+
153
+ try:
154
+ backend.listen([domain], callback)
155
+ except KeyboardInterrupt:
156
+ click.secho("\nStopping listener.", fg="yellow")
157
+ finally:
158
+ if backend.connection and backend.connection.is_open:
159
+ backend.connection.close()
streaming/api.py ADDED
File without changes
streaming/apps.py ADDED
@@ -0,0 +1,17 @@
1
+ import logging
2
+ from pathlib import Path
3
+
4
+ from django.apps import AppConfig
5
+
6
+ logger = logging.getLogger(__name__)
7
+
8
+
9
+ class StreamingConfig(AppConfig):
10
+ default_auto_field = "django.db.models.BigAutoField"
11
+ name = "streaming"
12
+ path = str(Path(__file__).parent)
13
+
14
+ def ready(self) -> None:
15
+ from .manager import initialize_engine
16
+
17
+ initialize_engine()
@@ -0,0 +1,33 @@
1
+ import logging
2
+ from typing import TYPE_CHECKING
3
+ from urllib.parse import urlparse
4
+
5
+ from streaming.exceptions import StreamingConfigError
6
+
7
+ if TYPE_CHECKING:
8
+ from ._base import BaseBackend
9
+
10
+ logger = logging.getLogger(__name__)
11
+
12
+
13
+ def get_backend() -> "BaseBackend":
14
+ from streaming.config import CONFIG
15
+
16
+ parsed_url = urlparse(CONFIG.BROKER_URL)
17
+ if parsed_url.scheme == "console":
18
+ from .console import ConsoleBackend
19
+
20
+ return ConsoleBackend(CONFIG.BROKER_URL)
21
+ if parsed_url.scheme == "redis":
22
+ from .redis import RedisBackend
23
+
24
+ return RedisBackend(CONFIG.BROKER_URL)
25
+ if parsed_url.scheme == "rabbit":
26
+ from .rabbitmq import RabbitMQBackend
27
+
28
+ return RabbitMQBackend(CONFIG.BROKER_URL)
29
+ if parsed_url.scheme == "debug":
30
+ from .debug import DebugBackend
31
+
32
+ return DebugBackend(CONFIG.BROKER_URL)
33
+ raise StreamingConfigError(f"Broker not supported: {parsed_url.scheme}")
@@ -0,0 +1,26 @@
1
+ import logging
2
+ from typing import TYPE_CHECKING
3
+ from urllib.parse import ParseResult, parse_qs, urlparse
4
+
5
+ from streaming.config import DEFAULT_QUEUE_NAME
6
+
7
+ logger = logging.getLogger(__name__)
8
+
9
+ if TYPE_CHECKING:
10
+ from streaming.types import EventType
11
+
12
+ not_provided = object()
13
+
14
+
15
+ class BaseBackend:
16
+ def __init__(self, url: str) -> None:
17
+ self.connection_url: str = url
18
+ self._parsed_url: ParseResult = urlparse(self.connection_url)
19
+ self._options = {k: v[0] for k, v in parse_qs(self._parsed_url.query).items()}
20
+ self.queue_name: str = self._options.pop("queue", DEFAULT_QUEUE_NAME)
21
+
22
+ def initialize(self) -> None:
23
+ pass
24
+
25
+ def publish(self, message: "EventType") -> None:
26
+ raise NotImplementedError()
@@ -0,0 +1,20 @@
1
+ import logging
2
+ import sys
3
+ from typing import TYPE_CHECKING
4
+
5
+ from ._base import BaseBackend
6
+
7
+ logger = logging.getLogger(__name__)
8
+
9
+ if TYPE_CHECKING:
10
+ from ..types import EventType
11
+
12
+
13
+ class ConsoleBackend(BaseBackend):
14
+ def __init__(self, url: str) -> None:
15
+ super().__init__(url)
16
+ self.stream = self._options.get("streams", "stdout")
17
+
18
+ def publish(self, message: "EventType") -> None:
19
+ stream = getattr(sys, self.stream)
20
+ stream.write(f"{message}\n")
@@ -0,0 +1,18 @@
1
+ import logging
2
+ from typing import TYPE_CHECKING
3
+
4
+ from ._base import BaseBackend
5
+
6
+ logger = logging.getLogger(__name__)
7
+
8
+ if TYPE_CHECKING:
9
+ from ..types import EventType
10
+
11
+
12
+ class DebugBackend(BaseBackend):
13
+ def __init__(self, url: str) -> None:
14
+ super().__init__(url)
15
+ self.messages: list[EventType] = []
16
+
17
+ def publish(self, message: "EventType") -> None:
18
+ self.messages.append(message)
@@ -0,0 +1,120 @@
1
+ import atexit
2
+ import json
3
+ import logging
4
+ import socket
5
+ import time
6
+ from typing import TYPE_CHECKING
7
+
8
+ import pika.channel
9
+ import pika.exceptions
10
+ from pika.exchange_type import ExchangeType
11
+
12
+ from streaming.config import CONFIG
13
+
14
+ from ..utils import DAY
15
+ from ._base import BaseBackend
16
+
17
+ if TYPE_CHECKING:
18
+ from pika.adapters.blocking_connection import BlockingChannel
19
+ from pika.spec import Basic, BasicProperties
20
+
21
+ from streaming.types import EventType, PikaCallback
22
+
23
+ logger = logging.getLogger(__name__)
24
+
25
+
26
+ class RabbitMQBackend(BaseBackend):
27
+ def __init__(self, url: str) -> None:
28
+ super().__init__(url)
29
+ self.host = str(self._parsed_url.hostname)
30
+ self.port = int(self._parsed_url.port) if self._parsed_url.port else 5672
31
+
32
+ self.exchange = self._options.get("exchange", "django-streaming-broadcast")
33
+ self.connection_name = self._options.get("connection_name", "")
34
+ self.timeout = float(self._options.get("timeout", 0.5))
35
+ self.routing_key = self._options.get("routing_key", "")
36
+ self.virtual_host = self._options.get("virtual_host", "/")
37
+
38
+ self.connection: pika.BlockingConnection | None = None
39
+ self.channel: BlockingChannel | None = None
40
+
41
+ atexit.register(self.close)
42
+
43
+ def connect(self) -> None:
44
+ logger.debug("Connecting to %s:%s", self.host, self.port)
45
+ if self.connection and self.connection.is_open:
46
+ self.close()
47
+
48
+ for __ in range(CONFIG.RETRY_COUNT):
49
+ try:
50
+ self.connection = pika.BlockingConnection(
51
+ pika.ConnectionParameters(
52
+ host=self.host,
53
+ port=self.port,
54
+ virtual_host=self.virtual_host,
55
+ socket_timeout=self.timeout,
56
+ blocked_connection_timeout=self.timeout,
57
+ stack_timeout=self.timeout,
58
+ client_properties={
59
+ "connection_name": self.connection_name,
60
+ "product": "django-streaming",
61
+ "information": "",
62
+ "version": "1.0",
63
+ },
64
+ )
65
+ )
66
+ self.channel = self.connection.channel()
67
+ self.channel.exchange_declare(exchange=self.exchange, exchange_type=ExchangeType.topic, durable=True)
68
+ self.channel.queue_declare(queue="global_queue", durable=True)
69
+ self.channel.queue_bind(exchange=self.exchange, queue="global_queue", routing_key="#")
70
+ return
71
+ except (socket.gaierror, pika.exceptions.AMQPConnectionError):
72
+ logger.warning(
73
+ f"Could not connect to RabbitMQ. Retrying in {CONFIG.RETRY_DELAY} seconds...",
74
+ )
75
+ time.sleep(CONFIG.RETRY_DELAY)
76
+ logger.critical("Could not connect to RabbitMQ after multiple retries.")
77
+
78
+ def listen(self, domains: list[str], callback: "PikaCallback") -> None:
79
+ def _callback(
80
+ ch: "BlockingChannel", method: "Basic.Deliver", properties: "BasicProperties", body: bytes
81
+ ) -> None:
82
+ callback(ch, method, properties, body)
83
+ ch.basic_ack(delivery_tag=method.delivery_tag) # type: ignore[arg-type]
84
+
85
+ if self.channel is None:
86
+ self.connect()
87
+
88
+ for domain in domains:
89
+ queue_name = f"{self.connection_name.lower()}_sub_to_{domain}"
90
+ self.channel.queue_declare(queue=queue_name, durable=True) # type: ignore[union-attr]
91
+ self.channel.queue_bind(exchange=self.exchange, queue=queue_name, routing_key=domain) # type: ignore[union-attr]
92
+ self.channel.basic_consume(queue=queue_name, on_message_callback=_callback, auto_ack=False) # type: ignore[union-attr]
93
+ self.channel.start_consuming() # type: ignore[union-attr]
94
+
95
+ def publish(self, message: "EventType") -> None:
96
+ if not self.channel or self.channel.is_closed:
97
+ self.connect()
98
+ if self.channel:
99
+ logger.debug("publish to %s %s", self.exchange, message.get("domain", ""))
100
+ try:
101
+ self.channel.basic_publish(
102
+ exchange=self.exchange,
103
+ routing_key=message.get("domain", self.routing_key) or self.routing_key,
104
+ body=json.dumps(message).encode(),
105
+ properties=pika.BasicProperties(
106
+ delivery_mode=pika.spec.PERSISTENT_DELIVERY_MODE,
107
+ expiration=str(DAY * 2 * 1000), # milliseconds
108
+ ),
109
+ )
110
+ except Exception as e: # noqa: BLE001
111
+ logger.critical("Unhandled error sending to RabbitMQ. Message not published.", exc_info=e)
112
+ else:
113
+ logger.critical("RabbitMQ connection not available after reconnect. Message not published.")
114
+
115
+ def close(self) -> None:
116
+ if self.connection and self.connection.is_open:
117
+ logger.debug("Closing RabbitMQ connection.")
118
+ self.connection.close()
119
+ self.connection = None
120
+ self.channel = None
@@ -0,0 +1,56 @@
1
+ import json
2
+ import logging
3
+ import time
4
+ from typing import TYPE_CHECKING
5
+
6
+ import redis
7
+ from redis.exceptions import ConnectionError as RedisConnectionError
8
+
9
+ from streaming.config import CONFIG
10
+ from streaming.exceptions import StreamingBackendError
11
+
12
+ from ._base import BaseBackend
13
+
14
+ if TYPE_CHECKING:
15
+ from redis import Redis
16
+
17
+ from ..types import EventType
18
+
19
+ logger = logging.getLogger(__name__)
20
+
21
+
22
+ class RedisBackend(BaseBackend):
23
+ redis_client: "Redis[bytes]"
24
+
25
+ def __init__(self, url: str) -> None:
26
+ super().__init__(url)
27
+ self.db = int(self._parsed_url.path.lstrip("/") or 0)
28
+ self.host = str(self._parsed_url.hostname)
29
+ self.port = int(self._parsed_url.port) if self._parsed_url.port else 6379
30
+ self.timeout = float(self._options.get("timeout", 0.5))
31
+
32
+ self.redis_client = self._get_client()
33
+
34
+ def _get_client(self) -> "Redis[bytes]":
35
+ for __ in range(CONFIG.RETRY_COUNT):
36
+ try:
37
+ client = redis.Redis(
38
+ host=self.host,
39
+ port=self.port,
40
+ db=self.db,
41
+ socket_timeout=self.timeout,
42
+ socket_connect_timeout=self.timeout,
43
+ )
44
+ client.ping()
45
+ return client
46
+ except RedisConnectionError:
47
+ logger.warning("Could not connect to Redis. Retrying in %s seconds...", CONFIG.RETRY_DELAY)
48
+ time.sleep(CONFIG.RETRY_DELAY)
49
+ raise StreamingBackendError("Could not connect to Redis after multiple retries.")
50
+
51
+ def publish(self, message: "EventType") -> None:
52
+ try:
53
+ self.redis_client.publish(self.queue_name, json.dumps(message).encode())
54
+ except RedisConnectionError:
55
+ self.redis_client = self._get_client()
56
+ self.redis_client.publish(self.queue_name, json.dumps(message).encode())
streaming/config.py ADDED
@@ -0,0 +1,42 @@
1
+ import logging
2
+ from collections import ChainMap
3
+ from typing import Any
4
+
5
+ from django.conf import settings
6
+ from django.core.signals import setting_changed
7
+ from django.dispatch import receiver
8
+
9
+ logger = logging.getLogger(__name__)
10
+
11
+ DEFAULT_QUEUE_NAME = "django_model_changes"
12
+
13
+
14
+ class StreamingConfig:
15
+ _DEFAULTS = {
16
+ "BROKER_URL": "redis://localhost:6379/0?queue=hope_changes",
17
+ "RETRY_COUNT": 3,
18
+ "RETRY_DELAY": 1,
19
+ "MANAGER_CLASS": "streaming.manager.ChangeManager",
20
+ }
21
+
22
+ def __init__(self) -> None:
23
+ self._overrides = getattr(settings, "STREAMING", {})
24
+ self._parsed = ChainMap(self._overrides, self._DEFAULTS)
25
+ self._cached: dict[str, Any] = {}
26
+
27
+ def __getattr__(self, name: str) -> Any:
28
+ if name not in self._parsed:
29
+ raise AttributeError(f" 'StreamingConfig' object has no attribute '{name}'")
30
+ if name not in self._cached:
31
+ self._cached[name] = self._parsed[name]
32
+ return self._cached[name]
33
+
34
+
35
+ CONFIG = StreamingConfig()
36
+
37
+
38
+ @receiver(setting_changed)
39
+ def reload_config(sender: Any, setting: str, **kwargs: Any) -> None:
40
+ global CONFIG # noqa: PLW0603
41
+ if setting == "STREAMING":
42
+ CONFIG = StreamingConfig()
@@ -0,0 +1,15 @@
1
+ import logging
2
+
3
+ logger = logging.getLogger(__name__)
4
+
5
+
6
+ class StreamingError(Exception):
7
+ pass
8
+
9
+
10
+ class StreamingConfigError(StreamingError):
11
+ pass
12
+
13
+
14
+ class StreamingBackendError(StreamingError):
15
+ pass
streaming/handlers.py ADDED
@@ -0,0 +1,3 @@
1
+ import logging
2
+
3
+ logger = logging.getLogger(__name__)
streaming/manager.py ADDED
@@ -0,0 +1,91 @@
1
+ import atexit
2
+ import logging
3
+ import queue
4
+ import threading
5
+ from typing import TYPE_CHECKING, Any
6
+
7
+ from django.db.models import Model
8
+ from django.db.models.signals import post_save
9
+ from django.utils.module_loading import import_string
10
+
11
+ from streaming.backends import get_backend
12
+
13
+ if TYPE_CHECKING:
14
+ from .types import EventType
15
+
16
+ logger = logging.getLogger(__name__)
17
+
18
+ not_provided = object()
19
+
20
+
21
+ class ChangeManager:
22
+ def __init__(self) -> None:
23
+ self._registry: set[type[Model]] = set()
24
+ self.backend = get_backend()
25
+
26
+ def register(self, model: type[Model], receiver: Any = None) -> None:
27
+ logger.debug("Registering %s", model)
28
+ self._registry.add(model)
29
+ post_save.connect(receiver or self._post_save_receiver, sender=model, weak=False)
30
+
31
+ def _post_save_receiver(self, sender: type[Model], instance: Model, created: bool, **kwargs: Any) -> None:
32
+ logger.debug("post_save event for %s", sender)
33
+ payload = {"model": sender.__name__, "pk": instance.pk, "created": created, "fields": {}}
34
+ for field in sender._meta.fields:
35
+ payload["fields"][field.name] = str(getattr(instance, field.name))
36
+ message: EventType = {"event": "post_save", "domain": sender._meta.app_label, "payload": payload}
37
+ self.notify(message)
38
+
39
+ def notify(self, event: "EventType") -> None:
40
+ logger.debug("notifying %s", event)
41
+ self.backend.publish(event)
42
+
43
+
44
+ class ThreadedChangeManager(ChangeManager):
45
+ def __init__(self) -> None:
46
+ super().__init__()
47
+ self.queue: queue.Queue[Any] = queue.Queue()
48
+ self.thread: threading.Thread | None = None
49
+ self.shutdown_event = threading.Event()
50
+ atexit.register(self.stop)
51
+
52
+ def start(self) -> None:
53
+ if self.thread is None:
54
+ self.thread = threading.Thread(target=self._worker, daemon=True)
55
+ self.thread.start()
56
+
57
+ def stop(self) -> None:
58
+ logger.info("Shutting down threaded change manager...")
59
+ self.shutdown_event.set()
60
+ if self.thread and self.thread.is_alive():
61
+ self.queue.join()
62
+ self.thread.join(timeout=5)
63
+
64
+ def _worker(self) -> None:
65
+ while not self.shutdown_event.is_set() or not self.queue.empty():
66
+ try:
67
+ message = self.queue.get(timeout=1)
68
+ self.backend.publish(message)
69
+ self.queue.task_done()
70
+ except queue.Empty:
71
+ continue
72
+
73
+ def notify(self, event: Any) -> None:
74
+ self.queue.put(event)
75
+ self.start()
76
+
77
+
78
+ def get_manager() -> ChangeManager:
79
+ from streaming.config import CONFIG
80
+
81
+ return import_string(CONFIG.MANAGER_CLASS)() # type: ignore[no-any-return]
82
+
83
+
84
+ def initialize_engine(reset: bool = False) -> ChangeManager | ThreadedChangeManager:
85
+ global manager # noqa: PLW0603
86
+ if manager is None or reset:
87
+ manager = get_manager()
88
+ return manager
89
+
90
+
91
+ manager: "ChangeManager | None" = None
streaming/py.typed ADDED
File without changes
streaming/types.pyi ADDED
@@ -0,0 +1,15 @@
1
+ from collections.abc import Callable
2
+ from typing import TypedDict
3
+
4
+ from pika.adapters.blocking_connection import BlockingChannel
5
+ from pika.spec import Basic, BasicProperties
6
+
7
+ JsonT = None | bool | int | float | str | list["JsonT"] | dict[str, "JsonT"]
8
+ JSON = dict[str, JsonT]
9
+
10
+ PikaCallback = Callable[[BlockingChannel, Basic.Deliver, BasicProperties, bytes], None]
11
+
12
+ class EventType(TypedDict):
13
+ event: str
14
+ domain: str
15
+ payload: JSON
streaming/utils.py ADDED
@@ -0,0 +1,22 @@
1
+ from typing import TYPE_CHECKING, Any
2
+
3
+ if TYPE_CHECKING:
4
+ from streaming.types import JSON, EventType
5
+
6
+ MINUTE = 60
7
+ HOUR = MINUTE * 60
8
+ DAY = HOUR * 24
9
+
10
+
11
+ def parse_bool(value: Any) -> bool:
12
+ if isinstance(value, str):
13
+ return value.lower() in ("yes", "true", "1", "y", "t")
14
+ return value in [1, True]
15
+
16
+
17
+ def make_event(message: "str | JSON", *, event: str = "", domain: str = "") -> "EventType":
18
+ if isinstance(message, str):
19
+ payload: JSON = {"message": message}
20
+ else:
21
+ payload = message
22
+ return {"event": event, "domain": domain, "payload": payload}
streaming/version.py ADDED
@@ -0,0 +1,34 @@
1
+ # file generated by setuptools-scm
2
+ # don't change, don't track in version control
3
+
4
+ __all__ = [
5
+ "__version__",
6
+ "__version_tuple__",
7
+ "version",
8
+ "version_tuple",
9
+ "__commit_id__",
10
+ "commit_id",
11
+ ]
12
+
13
+ TYPE_CHECKING = False
14
+ if TYPE_CHECKING:
15
+ from typing import Tuple
16
+ from typing import Union
17
+
18
+ VERSION_TUPLE = Tuple[Union[int, str], ...]
19
+ COMMIT_ID = Union[str, None]
20
+ else:
21
+ VERSION_TUPLE = object
22
+ COMMIT_ID = object
23
+
24
+ version: str
25
+ __version__: str
26
+ __version_tuple__: VERSION_TUPLE
27
+ version_tuple: VERSION_TUPLE
28
+ commit_id: COMMIT_ID
29
+ __commit_id__: COMMIT_ID
30
+
31
+ __version__ = version = '0.1'
32
+ __version_tuple__ = version_tuple = (0, 1)
33
+
34
+ __commit_id__ = commit_id = None