django-streaming 0.3__tar.gz → 0.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. {django_streaming-0.3 → django_streaming-0.5}/PKG-INFO +7 -9
  2. {django_streaming-0.3 → django_streaming-0.5}/README.md +1 -1
  3. {django_streaming-0.3 → django_streaming-0.5}/pyproject.toml +6 -10
  4. django_streaming-0.5/src/streaming/__cli__.py +198 -0
  5. {django_streaming-0.3 → django_streaming-0.5}/src/streaming/apps.py +0 -5
  6. {django_streaming-0.3 → django_streaming-0.5}/src/streaming/backends/__init__.py +9 -6
  7. django_streaming-0.5/src/streaming/backends/_base.py +87 -0
  8. django_streaming-0.5/src/streaming/backends/console.py +30 -0
  9. django_streaming-0.5/src/streaming/backends/debug.py +28 -0
  10. django_streaming-0.5/src/streaming/backends/rabbitmq.py +225 -0
  11. {django_streaming-0.3 → django_streaming-0.5}/src/streaming/config.py +14 -8
  12. django_streaming-0.5/src/streaming/exceptions.py +27 -0
  13. {django_streaming-0.3 → django_streaming-0.5}/src/streaming/manager.py +15 -10
  14. django_streaming-0.5/src/streaming/py.typed +0 -0
  15. {django_streaming-0.3 → django_streaming-0.5}/src/streaming/types.pyi +7 -3
  16. django_streaming-0.5/src/streaming/utils.py +78 -0
  17. {django_streaming-0.3 → django_streaming-0.5}/src/streaming/version.py +2 -2
  18. django_streaming-0.3/src/streaming/__cli__.py +0 -159
  19. django_streaming-0.3/src/streaming/backends/_base.py +0 -26
  20. django_streaming-0.3/src/streaming/backends/console.py +0 -20
  21. django_streaming-0.3/src/streaming/backends/debug.py +0 -18
  22. django_streaming-0.3/src/streaming/backends/rabbitmq.py +0 -127
  23. django_streaming-0.3/src/streaming/backends/redis.py +0 -56
  24. django_streaming-0.3/src/streaming/exceptions.py +0 -15
  25. django_streaming-0.3/src/streaming/handlers.py +0 -3
  26. django_streaming-0.3/src/streaming/utils.py +0 -22
  27. {django_streaming-0.3 → django_streaming-0.5}/.gitignore +0 -0
  28. {django_streaming-0.3 → django_streaming-0.5}/LICENSE +0 -0
  29. /django_streaming-0.3/src/streaming/api.py → /django_streaming-0.5/src/streaming/__init__.py +0 -0
  30. /django_streaming-0.3/src/streaming/py.typed → /django_streaming-0.5/src/streaming/api.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: django-streaming
3
- Version: 0.3
3
+ Version: 0.5
4
4
  Summary: A project for streaming data.
5
5
  Project-URL: Documentation, https://github.com/UNICEF/django-streaming
6
6
  Project-URL: Homepage, https://github.com/UNICEF/django-streaming
@@ -8,29 +8,27 @@ Author-email: Stefano Apostolico <s.apostolico@gmail.com>
8
8
  License-File: LICENSE
9
9
  Classifier: Environment :: Web Environment
10
10
  Classifier: Framework :: Django
11
+ Classifier: Framework :: Django :: 3.2
11
12
  Classifier: Framework :: Django :: 4.2
12
13
  Classifier: Framework :: Django :: 5.2
13
14
  Classifier: Intended Audience :: Developers
14
15
  Classifier: License :: OSI Approved :: MIT License
15
16
  Classifier: Operating System :: OS Independent
16
17
  Classifier: Programming Language :: Python :: 3 :: Only
17
- Classifier: Programming Language :: Python :: 3.9
18
18
  Classifier: Programming Language :: Python :: 3.10
19
19
  Classifier: Programming Language :: Python :: 3.11
20
20
  Classifier: Programming Language :: Python :: 3.12
21
21
  Classifier: Programming Language :: Python :: 3.13
22
22
  Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
23
23
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
24
- Requires-Python: >=3.9
24
+ Requires-Python: >=3.10
25
25
  Requires-Dist: click>=8
26
+ Requires-Dist: colorama>=0.4.6
26
27
  Requires-Dist: django-stubs-ext
27
- Requires-Dist: django>=4
28
+ Requires-Dist: django>=3
28
29
  Requires-Dist: markdown>=3.8.2
30
+ Requires-Dist: pika>=1.3.2
29
31
  Requires-Dist: requests>=2.32.5
30
- Provides-Extra: rabbitmq
31
- Requires-Dist: pika>=1.3.2; extra == 'rabbitmq'
32
- Provides-Extra: redis
33
- Requires-Dist: redis>=5; extra == 'redis'
34
32
  Description-Content-Type: text/markdown
35
33
 
36
34
  # Django Streaming
@@ -45,7 +43,7 @@ Description-Content-Type: text/markdown
45
43
  `django-streaming` is a robust and flexible Django application designed for efficiently streaming data from your
46
44
  Django projects to various message brokers.
47
45
 
48
- It provides a pluggable backend architecture supporting popular options like RabbitMQ and Redis, alongside console and
46
+ It provides a pluggable backend architecture supporting popular options like RabbitMQ, alongside console and
49
47
  debug backends for development. With features like a threaded change manager for non-blocking asynchronous
50
48
  notifications, configurable connection retry mechanisms, and graceful shutdown handling, django-streaming ensures
51
49
  reliable data delivery. It seamlessly integrates with Django's ORM via signals to automatically stream model changes
@@ -10,7 +10,7 @@
10
10
  `django-streaming` is a robust and flexible Django application designed for efficiently streaming data from your
11
11
  Django projects to various message brokers.
12
12
 
13
- It provides a pluggable backend architecture supporting popular options like RabbitMQ and Redis, alongside console and
13
+ It provides a pluggable backend architecture supporting popular options like RabbitMQ, alongside console and
14
14
  debug backends for development. With features like a threaded change manager for non-blocking asynchronous
15
15
  notifications, configurable connection retry mechanisms, and graceful shutdown handling, django-streaming ensures
16
16
  reliable data delivery. It seamlessly integrates with Django's ORM via signals to automatically stream model changes
@@ -12,17 +12,17 @@ readme = "README.md"
12
12
  authors = [
13
13
  { name = "Stefano Apostolico", email = "s.apostolico@gmail.com" },
14
14
  ]
15
- requires-python = ">=3.9"
15
+ requires-python = ">=3.10"
16
16
  classifiers = [
17
17
  "Environment :: Web Environment",
18
18
  "Framework :: Django",
19
+ "Framework :: Django :: 3.2",
19
20
  "Framework :: Django :: 4.2",
20
21
  "Framework :: Django :: 5.2",
21
22
  "Intended Audience :: Developers",
22
23
  "License :: OSI Approved :: MIT License",
23
24
  "Operating System :: OS Independent",
24
25
  "Programming Language :: Python :: 3 :: Only",
25
- "Programming Language :: Python :: 3.9",
26
26
  "Programming Language :: Python :: 3.10",
27
27
  "Programming Language :: Python :: 3.11",
28
28
  "Programming Language :: Python :: 3.12",
@@ -35,18 +35,14 @@ dynamic = [
35
35
  ]
36
36
  dependencies = [
37
37
  "click>=8",
38
- "django>=4",
38
+ "colorama>=0.4.6",
39
+ "django>=3",
39
40
  "django-stubs-ext",
40
41
  "markdown>=3.8.2",
41
- "requests>=2.32.5",
42
- ]
43
- optional-dependencies.rabbitmq = [
44
42
  "pika>=1.3.2",
43
+ "requests>=2.32.5",
45
44
  ]
46
45
 
47
- optional-dependencies.redis = [
48
- "redis>=5",
49
- ]
50
46
  urls.Documentation = "https://github.com/UNICEF/django-streaming"
51
47
  urls.Homepage = "https://github.com/UNICEF/django-streaming"
52
48
 
@@ -57,6 +53,7 @@ dev = [
57
53
  "django-webtest>=1.9.13",
58
54
  "factory-boy>=3.3.3",
59
55
  "pre-commit>=3.3.3",
56
+ "pyrabbit2>=1.0.7",
60
57
  "pytest>=8.4.1",
61
58
  "pytest-cov>=6.2.1",
62
59
  "pytest-django>=4.11.1",
@@ -94,7 +91,6 @@ type-check = [
94
91
  "mypy",
95
92
  "types-markdown",
96
93
  "types-pika",
97
- "types-redis",
98
94
  "types-requests",
99
95
  ]
100
96
 
@@ -0,0 +1,198 @@
1
+ import json
2
+ import logging
3
+ from typing import TYPE_CHECKING
4
+
5
+ import click
6
+ from click import ClickException
7
+ from colorama import Fore, Style
8
+ from django.core.exceptions import ImproperlyConfigured
9
+ from pika.adapters.blocking_connection import BlockingChannel
10
+ from pika.exceptions import ChannelClosedByBroker
11
+ from pika.spec import Basic, BasicProperties
12
+
13
+ from .backends import RabbitMQBackend, get_backend
14
+ from .exceptions import AuthorizationError, StreamingConfigError
15
+ from .utils import make_event
16
+
17
+ if TYPE_CHECKING:
18
+ from .types import JSON, EventType
19
+
20
+
21
+ logger = logging.getLogger(__name__)
22
+
23
+
24
+ def _dump_info(backend: "RabbitMQBackend") -> None:
25
+ from streaming.config import CONFIG
26
+
27
+ line = f"{Fore.YELLOW}%-16s: {Style.RESET_ALL}%s"
28
+ click.secho(line % ("Server", f"{backend.host}:{backend.port}"))
29
+ click.secho(line % ("VirtualHost", f"{backend.virtual_host}"))
30
+ click.secho(line % ("Exchange", f"{backend.exchange}"))
31
+ click.secho(line % ("Queues", ""))
32
+ for alias, config in CONFIG.QUEUES.items():
33
+ click.secho(line % (f" {alias}", f"{config}"))
34
+ click.secho(line % ("Timeout", f"{backend.timeout}"))
35
+ click.secho(line % ("Client Name", f"{backend.client_name}"))
36
+
37
+
38
+ def assert_backend() -> "RabbitMQBackend":
39
+ backend: RabbitMQBackend = get_backend() # type: ignore[assignment]
40
+
41
+ if not isinstance(backend, RabbitMQBackend):
42
+ raise ClickException("RabbitMQ backend is not configured")
43
+ return backend
44
+
45
+
46
+ @click.group()
47
+ def cli() -> None:
48
+ """Streaming CLI."""
49
+ try:
50
+ import django
51
+
52
+ django.setup()
53
+ except ModuleNotFoundError as e:
54
+ raise ClickException(f"Unable to setup Django. {e}") from e
55
+ except ImproperlyConfigured as e:
56
+ raise ClickException("Unable to setup Django. Is DJANGO_SETTINGS_MODULE environment variable set?") from e
57
+
58
+
59
+ @cli.command()
60
+ @click.option("--client-name", default=None, help="Override client name")
61
+ def configure(client_name: str) -> None:
62
+ from streaming.config import CONFIG
63
+
64
+ backend: RabbitMQBackend = assert_backend()
65
+ if client_name:
66
+ backend.client_name = client_name
67
+ try:
68
+ backend.connect(True)
69
+ backend.configure_exchanges()
70
+ backend.configure_queue_routing()
71
+ check.callback() # type: ignore[misc]
72
+ except AuthorizationError as e:
73
+ click.secho(f"Unable to connect using {CONFIG.BROKER_URL}", fg="red", err=True)
74
+ raise ClickException(str(e)) from e
75
+ except StreamingConfigError as e:
76
+ click.secho(f"Generic error {e}", fg="red", err=True)
77
+ raise ClickException(str(e)) from e
78
+
79
+
80
+ @cli.command()
81
+ @click.argument("routing_key")
82
+ @click.option("-c", "--client-name", default=None, help="Override client name")
83
+ @click.option("--message", default="Test Message", help="Message to send")
84
+ @click.option("--debug", is_flag=True, help="Debug mode")
85
+ def send(routing_key: str, message: str, client_name: str, debug: bool) -> None:
86
+ backend = assert_backend()
87
+ logger = logging.getLogger("streaming")
88
+ if debug:
89
+ logger.setLevel(logging.DEBUG)
90
+ logger.addHandler(logging.StreamHandler())
91
+ else:
92
+ logger.handlers = []
93
+
94
+ if client_name:
95
+ backend.client_name = client_name
96
+ try:
97
+ payload = json.loads(message)
98
+ except json.decoder.JSONDecodeError:
99
+ payload = {
100
+ "message": message,
101
+ }
102
+ msg: EventType = make_event(payload, event="Test")
103
+ backend.publish(routing_key, msg)
104
+ click.secho(f"Sent: {msg}")
105
+ backend.disconnect()
106
+
107
+
108
+ def _listen(queues: list[str], payload: bool, pretty: bool, client_name: str) -> None:
109
+ backend = assert_backend()
110
+
111
+ if client_name:
112
+ backend.client_name = client_name
113
+
114
+ def callback(
115
+ queue_name: str, ch: BlockingChannel, method: Basic.Deliver, properties: BasicProperties, body: bytes
116
+ ) -> None:
117
+ message: EventType = json.loads(body.decode())
118
+ click.echo(
119
+ f"{Fore.GREEN}{message['timestamp']} "
120
+ f"[{queue_name}]"
121
+ f"{Fore.LIGHTWHITE_EX} [{message['type']}]"
122
+ f"{message['event']} "
123
+ )
124
+ extra: str | JSON
125
+ if payload:
126
+ if pretty:
127
+ extra = json.dumps(message["payload"], indent=4)
128
+ else:
129
+ extra = message["payload"]
130
+ click.echo(f"{Fore.YELLOW}{extra}{Fore.RESET}")
131
+
132
+ try:
133
+ backend.connect()
134
+ _dump_info(backend)
135
+ backend.listen(callback, queues=queues)
136
+ except KeyboardInterrupt:
137
+ click.secho("Stopping listener.", fg="yellow")
138
+ finally:
139
+ backend.disconnect()
140
+
141
+
142
+ @cli.command()
143
+ @click.option("-q", "--queues", multiple=True, help="Queue name to listen to")
144
+ @click.option("-c", "--client-name", default=None, help="Override client name")
145
+ @click.option("--payload", default=False, is_flag=True, help="Print payload")
146
+ @click.option("--autoreload", "reload", is_flag=True, help="Enable auto-reloading.")
147
+ @click.option("--pretty", is_flag=True, help="Pretty-print payload.")
148
+ def listen(queues: list[str], payload: bool, reload: bool, pretty: bool, client_name: str) -> None:
149
+ """Listens for streaming events."""
150
+ if reload:
151
+ from django.utils import autoreload
152
+
153
+ click.secho("Starting listener with autoreload...", fg="yellow")
154
+ autoreload.run_with_reloader(_listen, queues=queues, payload=payload, pretty=pretty, client_name=client_name)
155
+ else:
156
+ _listen(queues=queues, payload=payload, pretty=pretty, client_name=client_name)
157
+
158
+
159
+ @cli.command()
160
+ def purge() -> None:
161
+ """Purges all messages from the configured queues."""
162
+ from streaming.backends.rabbitmq import RabbitMQBackend
163
+ from streaming.config import CONFIG
164
+ from streaming.manager import initialize_engine
165
+
166
+ manager = initialize_engine(True)
167
+ backend = manager.backend
168
+ if not isinstance(backend, RabbitMQBackend):
169
+ raise click.ClickException("RabbitMQ backend is not configured. Please set BROKER_URL to a rabbit:// URL.")
170
+
171
+ backend.connect(True)
172
+ for queue_alias, queue_config in CONFIG.QUEUES.items():
173
+ queue_name = queue_config.get("name", queue_alias)
174
+ try:
175
+ message_count = backend.channel.queue_purge(queue_name) # type: ignore[union-attr]
176
+ click.secho(f"Purged {message_count.method.message_count} messages from queue '{queue_name}'.", fg="green")
177
+ except ChannelClosedByBroker:
178
+ click.secho(f"Could not purge queue '{queue_name}'. Queue may not exist.", fg="red")
179
+ backend.disconnect()
180
+
181
+
182
+ @cli.command()
183
+ def check() -> None:
184
+ """Checks the streaming configuration and connection."""
185
+ from streaming.config import CONFIG
186
+
187
+ click.secho("System Configuration:")
188
+ config_dict = dict(CONFIG._parsed)
189
+ for key, value in config_dict.items():
190
+ click.echo(f" {key}: {value}")
191
+
192
+ backend: RabbitMQBackend = assert_backend()
193
+ try:
194
+ backend.connect(True)
195
+ click.secho("Connection successful.", fg="green")
196
+ _dump_info(backend)
197
+ except (StreamingConfigError, AuthorizationError) as e:
198
+ raise ClickException(f"Connection failed: {e}") from e
@@ -10,8 +10,3 @@ class StreamingConfig(AppConfig):
10
10
  default_auto_field = "django.db.models.BigAutoField"
11
11
  name = "streaming"
12
12
  path = str(Path(__file__).parent)
13
-
14
- def ready(self) -> None:
15
- from .manager import initialize_engine
16
-
17
- initialize_engine()
@@ -2,26 +2,29 @@ import logging
2
2
  from typing import TYPE_CHECKING
3
3
  from urllib.parse import urlparse
4
4
 
5
- from streaming.exceptions import StreamingConfigError
5
+ from ..exceptions import StreamingConfigError
6
+ from .rabbitmq import RabbitMQBackend
6
7
 
7
8
  if TYPE_CHECKING:
8
9
  from ._base import BaseBackend
9
10
 
10
11
  logger = logging.getLogger(__name__)
11
12
 
13
+ __all__ = ["RabbitMQBackend", "get_backend"]
14
+
12
15
 
13
16
  def get_backend() -> "BaseBackend":
14
17
  from streaming.config import CONFIG
15
18
 
19
+ if not CONFIG.BROKER_URL:
20
+ raise StreamingConfigError("Empty BROKER_URL")
21
+
16
22
  parsed_url = urlparse(CONFIG.BROKER_URL)
23
+
17
24
  if parsed_url.scheme == "console":
18
25
  from .console import ConsoleBackend
19
26
 
20
27
  return ConsoleBackend(CONFIG.BROKER_URL)
21
- if parsed_url.scheme == "redis":
22
- from .redis import RedisBackend
23
-
24
- return RedisBackend(CONFIG.BROKER_URL)
25
28
  if parsed_url.scheme == "rabbit":
26
29
  from .rabbitmq import RabbitMQBackend
27
30
 
@@ -30,4 +33,4 @@ def get_backend() -> "BaseBackend":
30
33
  from .debug import DebugBackend
31
34
 
32
35
  return DebugBackend(CONFIG.BROKER_URL)
33
- raise StreamingConfigError(f"Broker not supported: {parsed_url.scheme}")
36
+ raise StreamingConfigError(f"Broker not supported: '{parsed_url.scheme}://{parsed_url.hostname}:{parsed_url.port}'")
@@ -0,0 +1,87 @@
1
+ import atexit
2
+ import logging
3
+ from abc import ABC, abstractmethod
4
+ from typing import TYPE_CHECKING, Any
5
+ from urllib.parse import ParseResult, parse_qs, urlparse
6
+
7
+ from streaming.config import CONFIG
8
+ from streaming.utils import get_local_ip
9
+
10
+ from ..exceptions import (
11
+ StreamingError,
12
+ )
13
+
14
+ if TYPE_CHECKING:
15
+ from pika.adapters.blocking_connection import BlockingChannel, BlockingConnection
16
+
17
+ from streaming.types import EventType, UserCallback
18
+
19
+
20
+ logger = logging.getLogger(__name__)
21
+
22
+ MAX_RETRIES = 5
23
+
24
+
25
+ class BaseBackend(ABC):
26
+ def __init__(self, url: str) -> None:
27
+ self.connection_url: str = url
28
+ self._parsed_url: ParseResult = urlparse(self.connection_url)
29
+ self._options = {k: v[0] for k, v in parse_qs(self._parsed_url.query).items()}
30
+ self.exchange = self.get_option("exchange", "django-streaming-broadcast")
31
+
32
+ self.host = str(self._parsed_url.hostname)
33
+ self.port = int(self._parsed_url.port) if self._parsed_url.port else 5672
34
+ self._connection: BlockingConnection | None = None
35
+ self._username = self._parsed_url.username or "guest"
36
+ self._password = self._parsed_url.password or "guest"
37
+
38
+ self.channel: BlockingChannel | None = None
39
+ self.exchange = self.get_option("exchange", "django-streaming-broadcast")
40
+ self.retry_exchange = f"retry_{self.exchange}"
41
+ self.timeout = float(self.get_option("timeout", 0.5))
42
+ self.virtual_host = self.get_option("vhost", "/")
43
+ # listener
44
+ self.client_name = CONFIG.CLIENT_NAME or get_local_ip()
45
+ atexit.register(self.disconnect)
46
+
47
+ def get_option(self, name: str, default: Any = "") -> Any:
48
+ return self._options.get(name, default)
49
+
50
+ @property
51
+ def client_name(self) -> str:
52
+ return self.__client_name
53
+
54
+ @client_name.setter
55
+ def client_name(self, name: str) -> None:
56
+ self.__client_name = name
57
+
58
+ def configure_exchanges(self) -> None: # noqa: B027
59
+ pass
60
+
61
+ def configure_queue_routing(self) -> None: # noqa: B027
62
+ pass
63
+
64
+ def get_real_queue_name(self, name: str) -> str:
65
+ return f"{self.client_name}:{name}"
66
+
67
+ def set_credential(self, username: str, password: str) -> None:
68
+ if self._connection:
69
+ raise StreamingError("Disconnect first.")
70
+ self.__username = username
71
+ self.__password = password
72
+
73
+ @abstractmethod
74
+ def connect(self, raise_if_error: bool = False) -> None:
75
+ pass
76
+
77
+ @abstractmethod
78
+ def disconnect(self) -> None:
79
+ pass
80
+
81
+ # Publisher
82
+ @abstractmethod
83
+ def publish(self, routing_key: str, message: "EventType") -> bool: ...
84
+
85
+ @abstractmethod
86
+ def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
87
+ pass
@@ -0,0 +1,30 @@
1
+ import logging
2
+ import sys
3
+ from typing import TYPE_CHECKING
4
+
5
+ from ._base import BaseBackend
6
+
7
+ logger = logging.getLogger(__name__)
8
+
9
+ if TYPE_CHECKING:
10
+ from ..types import EventType, UserCallback
11
+
12
+
13
+ class ConsoleBackend(BaseBackend):
14
+ def __init__(self, url: str) -> None:
15
+ super().__init__(url)
16
+ self.stream = self.get_option("stream", "stdout")
17
+
18
+ def publish(self, routing_key: str, message: "EventType") -> bool:
19
+ stream = getattr(sys, self.stream)
20
+ stream.write(f"routing_key:{routing_key} message:{message}\n")
21
+ return True
22
+
23
+ def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
24
+ pass
25
+
26
+ def connect(self, raise_if_error: bool = False) -> None:
27
+ pass
28
+
29
+ def disconnect(self) -> None:
30
+ pass
@@ -0,0 +1,28 @@
1
+ import logging
2
+ from typing import TYPE_CHECKING
3
+
4
+ from ._base import BaseBackend
5
+
6
+ logger = logging.getLogger(__name__)
7
+
8
+ if TYPE_CHECKING:
9
+ from ..types import EventType, UserCallback
10
+
11
+
12
+ class DebugBackend(BaseBackend):
13
+ def __init__(self, url: str) -> None:
14
+ super().__init__(url)
15
+ self.messages: list[tuple[str, EventType]] = []
16
+
17
+ def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
18
+ pass
19
+
20
+ def connect(self, raise_if_error: bool = False) -> None:
21
+ pass
22
+
23
+ def disconnect(self) -> None:
24
+ pass
25
+
26
+ def publish(self, routing_key: str, message: "EventType") -> bool:
27
+ self.messages.append((routing_key, message))
28
+ return True