django-streaming 0.5__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {django_streaming-0.5 → django_streaming-0.6.0}/PKG-INFO +1 -1
- {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/__cli__.py +47 -43
- {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/backends/_base.py +9 -6
- {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/backends/console.py +9 -2
- {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/backends/debug.py +10 -3
- {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/backends/rabbitmq.py +37 -30
- django_streaming-0.6.0/src/streaming/event.py +64 -0
- {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/exceptions.py +1 -1
- {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/manager.py +4 -4
- {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/types.pyi +2 -8
- {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/utils.py +7 -35
- {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/version.py +2 -2
- django_streaming-0.5/src/streaming/api.py +0 -0
- {django_streaming-0.5 → django_streaming-0.6.0}/.gitignore +0 -0
- {django_streaming-0.5 → django_streaming-0.6.0}/LICENSE +0 -0
- {django_streaming-0.5 → django_streaming-0.6.0}/README.md +0 -0
- {django_streaming-0.5 → django_streaming-0.6.0}/pyproject.toml +0 -0
- {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/__init__.py +0 -0
- {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/apps.py +0 -0
- {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/backends/__init__.py +0 -0
- {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/config.py +0 -0
- {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/py.typed +0 -0
|
@@ -11,28 +11,23 @@ from pika.exceptions import ChannelClosedByBroker
|
|
|
11
11
|
from pika.spec import Basic, BasicProperties
|
|
12
12
|
|
|
13
13
|
from .backends import RabbitMQBackend, get_backend
|
|
14
|
+
from .event import Event
|
|
14
15
|
from .exceptions import AuthorizationError, StreamingConfigError
|
|
15
16
|
from .utils import make_event
|
|
16
17
|
|
|
17
18
|
if TYPE_CHECKING:
|
|
18
|
-
from .types import JSON
|
|
19
|
-
|
|
19
|
+
from .types import JSON
|
|
20
20
|
|
|
21
21
|
logger = logging.getLogger(__name__)
|
|
22
|
+
LINE = f"{Fore.YELLOW}%-16s: {Style.RESET_ALL}%s"
|
|
22
23
|
|
|
23
24
|
|
|
24
25
|
def _dump_info(backend: "RabbitMQBackend") -> None:
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
click.secho(
|
|
29
|
-
click.secho(
|
|
30
|
-
click.secho(line % ("Exchange", f"{backend.exchange}"))
|
|
31
|
-
click.secho(line % ("Queues", ""))
|
|
32
|
-
for alias, config in CONFIG.QUEUES.items():
|
|
33
|
-
click.secho(line % (f" {alias}", f"{config}"))
|
|
34
|
-
click.secho(line % ("Timeout", f"{backend.timeout}"))
|
|
35
|
-
click.secho(line % ("Client Name", f"{backend.client_name}"))
|
|
26
|
+
click.secho(LINE % ("Server", f"{backend.host}:{backend.port}"))
|
|
27
|
+
click.secho(LINE % ("VirtualHost", f"{backend.virtual_host}"))
|
|
28
|
+
click.secho(LINE % ("Exchange", f"{backend.exchange}"))
|
|
29
|
+
click.secho(LINE % ("Timeout", f"{backend.timeout}"))
|
|
30
|
+
click.secho(LINE % ("Client Name", f"{backend.client_name}"))
|
|
36
31
|
|
|
37
32
|
|
|
38
33
|
def assert_backend() -> "RabbitMQBackend":
|
|
@@ -43,6 +38,16 @@ def assert_backend() -> "RabbitMQBackend":
|
|
|
43
38
|
return backend
|
|
44
39
|
|
|
45
40
|
|
|
41
|
+
def configure_logging(debug: bool, loggers: tuple[str, ...] = ("streaming",)) -> None:
|
|
42
|
+
for log_name in loggers:
|
|
43
|
+
logr = logging.getLogger(log_name)
|
|
44
|
+
if debug:
|
|
45
|
+
logr.setLevel(logging.DEBUG)
|
|
46
|
+
logr.addHandler(logging.StreamHandler())
|
|
47
|
+
else:
|
|
48
|
+
logr.handlers = []
|
|
49
|
+
|
|
50
|
+
|
|
46
51
|
@click.group()
|
|
47
52
|
def cli() -> None:
|
|
48
53
|
"""Streaming CLI."""
|
|
@@ -57,18 +62,24 @@ def cli() -> None:
|
|
|
57
62
|
|
|
58
63
|
|
|
59
64
|
@cli.command()
|
|
60
|
-
@click.option("--
|
|
61
|
-
|
|
65
|
+
@click.option("--queues/--no-queues", "queues", is_flag=True, default=False, help="Debug mode")
|
|
66
|
+
@click.option("--debug", is_flag=True, help="Debug mode")
|
|
67
|
+
def configure(queues: bool = False, debug: bool = False) -> None:
|
|
62
68
|
from streaming.config import CONFIG
|
|
63
69
|
|
|
70
|
+
routing = None
|
|
71
|
+
configure_logging(debug)
|
|
64
72
|
backend: RabbitMQBackend = assert_backend()
|
|
65
|
-
if client_name:
|
|
66
|
-
backend.client_name = client_name
|
|
67
73
|
try:
|
|
68
74
|
backend.connect(True)
|
|
69
75
|
backend.configure_exchanges()
|
|
70
|
-
|
|
76
|
+
if queues:
|
|
77
|
+
routing = backend.configure_queue_routing()
|
|
71
78
|
check.callback() # type: ignore[misc]
|
|
79
|
+
if routing:
|
|
80
|
+
click.secho(LINE % ("Queues", ""))
|
|
81
|
+
for k, v in routing.items():
|
|
82
|
+
click.secho(LINE % (f" {k}", "; ".join(v)))
|
|
72
83
|
except AuthorizationError as e:
|
|
73
84
|
click.secho(f"Unable to connect using {CONFIG.BROKER_URL}", fg="red", err=True)
|
|
74
85
|
raise ClickException(str(e)) from e
|
|
@@ -84,12 +95,7 @@ def configure(client_name: str) -> None:
|
|
|
84
95
|
@click.option("--debug", is_flag=True, help="Debug mode")
|
|
85
96
|
def send(routing_key: str, message: str, client_name: str, debug: bool) -> None:
|
|
86
97
|
backend = assert_backend()
|
|
87
|
-
|
|
88
|
-
if debug:
|
|
89
|
-
logger.setLevel(logging.DEBUG)
|
|
90
|
-
logger.addHandler(logging.StreamHandler())
|
|
91
|
-
else:
|
|
92
|
-
logger.handlers = []
|
|
98
|
+
configure_logging(debug)
|
|
93
99
|
|
|
94
100
|
if client_name:
|
|
95
101
|
backend.client_name = client_name
|
|
@@ -99,40 +105,39 @@ def send(routing_key: str, message: str, client_name: str, debug: bool) -> None:
|
|
|
99
105
|
payload = {
|
|
100
106
|
"message": message,
|
|
101
107
|
}
|
|
102
|
-
msg:
|
|
108
|
+
msg: Event = make_event(payload, key="Test")
|
|
103
109
|
backend.publish(routing_key, msg)
|
|
104
110
|
click.secho(f"Sent: {msg}")
|
|
105
111
|
backend.disconnect()
|
|
106
112
|
|
|
107
113
|
|
|
108
|
-
def _listen(queues: list[str], payload: bool, pretty: bool
|
|
114
|
+
def _listen(queues: list[str], payload: bool, pretty: bool) -> None:
|
|
109
115
|
backend = assert_backend()
|
|
110
116
|
|
|
111
|
-
if client_name:
|
|
112
|
-
backend.client_name = client_name
|
|
113
|
-
|
|
114
117
|
def callback(
|
|
115
118
|
queue_name: str, ch: BlockingChannel, method: Basic.Deliver, properties: BasicProperties, body: bytes
|
|
116
119
|
) -> None:
|
|
117
|
-
message:
|
|
118
|
-
click.echo(
|
|
119
|
-
f"{Fore.GREEN}{message['timestamp']} "
|
|
120
|
-
f"[{queue_name}]"
|
|
121
|
-
f"{Fore.LIGHTWHITE_EX} [{message['type']}]"
|
|
122
|
-
f"{message['event']} "
|
|
123
|
-
)
|
|
120
|
+
message: Event = Event.unmarshal(body)
|
|
121
|
+
click.echo(f"{Fore.GREEN}{message.timestamp} [{queue_name}]{Fore.LIGHTWHITE_EX} [{message.key}] {message.id} ")
|
|
124
122
|
extra: str | JSON
|
|
125
123
|
if payload:
|
|
126
124
|
if pretty:
|
|
127
|
-
extra = json.dumps(message
|
|
125
|
+
extra = json.dumps(message.payload, indent=4)
|
|
128
126
|
else:
|
|
129
|
-
extra = message
|
|
127
|
+
extra = message.payload
|
|
130
128
|
click.echo(f"{Fore.YELLOW}{extra}{Fore.RESET}")
|
|
131
129
|
|
|
132
130
|
try:
|
|
133
|
-
backend.connect()
|
|
131
|
+
backend.connect(True)
|
|
134
132
|
_dump_info(backend)
|
|
135
133
|
backend.listen(callback, queues=queues)
|
|
134
|
+
except ChannelClosedByBroker as e:
|
|
135
|
+
click.secho(str(e), fg="red")
|
|
136
|
+
if "no queue" in str(e):
|
|
137
|
+
click.secho("Did you run 'stream configure --queues'", fg="red")
|
|
138
|
+
except StreamingConfigError as e:
|
|
139
|
+
click.secho(str(e), fg="red")
|
|
140
|
+
click.get_current_context().exit(2)
|
|
136
141
|
except KeyboardInterrupt:
|
|
137
142
|
click.secho("Stopping listener.", fg="yellow")
|
|
138
143
|
finally:
|
|
@@ -141,19 +146,18 @@ def _listen(queues: list[str], payload: bool, pretty: bool, client_name: str) ->
|
|
|
141
146
|
|
|
142
147
|
@cli.command()
|
|
143
148
|
@click.option("-q", "--queues", multiple=True, help="Queue name to listen to")
|
|
144
|
-
@click.option("-c", "--client-name", default=None, help="Override client name")
|
|
145
149
|
@click.option("--payload", default=False, is_flag=True, help="Print payload")
|
|
146
150
|
@click.option("--autoreload", "reload", is_flag=True, help="Enable auto-reloading.")
|
|
147
151
|
@click.option("--pretty", is_flag=True, help="Pretty-print payload.")
|
|
148
|
-
def listen(queues: list[str], payload: bool, reload: bool, pretty: bool
|
|
152
|
+
def listen(queues: list[str], payload: bool, reload: bool, pretty: bool) -> None:
|
|
149
153
|
"""Listens for streaming events."""
|
|
150
154
|
if reload:
|
|
151
155
|
from django.utils import autoreload
|
|
152
156
|
|
|
153
157
|
click.secho("Starting listener with autoreload...", fg="yellow")
|
|
154
|
-
autoreload.run_with_reloader(_listen, queues=queues, payload=payload, pretty=pretty
|
|
158
|
+
autoreload.run_with_reloader(_listen, queues=queues, payload=payload, pretty=pretty)
|
|
155
159
|
else:
|
|
156
|
-
_listen(queues=queues, payload=payload, pretty=pretty
|
|
160
|
+
_listen(queues=queues, payload=payload, pretty=pretty)
|
|
157
161
|
|
|
158
162
|
|
|
159
163
|
@cli.command()
|
|
@@ -14,7 +14,8 @@ from ..exceptions import (
|
|
|
14
14
|
if TYPE_CHECKING:
|
|
15
15
|
from pika.adapters.blocking_connection import BlockingChannel, BlockingConnection
|
|
16
16
|
|
|
17
|
-
from
|
|
17
|
+
from ..event import Event
|
|
18
|
+
from ..types import UserCallback
|
|
18
19
|
|
|
19
20
|
|
|
20
21
|
logger = logging.getLogger(__name__)
|
|
@@ -41,7 +42,7 @@ class BaseBackend(ABC):
|
|
|
41
42
|
self.timeout = float(self.get_option("timeout", 0.5))
|
|
42
43
|
self.virtual_host = self.get_option("vhost", "/")
|
|
43
44
|
# listener
|
|
44
|
-
self.client_name = CONFIG.CLIENT_NAME or get_local_ip()
|
|
45
|
+
self.client_name = CONFIG.CLIENT_NAME or self.get_option("client_name", get_local_ip())
|
|
45
46
|
atexit.register(self.disconnect)
|
|
46
47
|
|
|
47
48
|
def get_option(self, name: str, default: Any = "") -> Any:
|
|
@@ -55,11 +56,13 @@ class BaseBackend(ABC):
|
|
|
55
56
|
def client_name(self, name: str) -> None:
|
|
56
57
|
self.__client_name = name
|
|
57
58
|
|
|
59
|
+
@abstractmethod
|
|
58
60
|
def configure_exchanges(self) -> None: # noqa: B027
|
|
59
|
-
|
|
61
|
+
...
|
|
60
62
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
+
@abstractmethod
|
|
64
|
+
def configure_queue_routing(self) -> dict[str, list[str]]: # noqa: B027
|
|
65
|
+
...
|
|
63
66
|
|
|
64
67
|
def get_real_queue_name(self, name: str) -> str:
|
|
65
68
|
return f"{self.client_name}:{name}"
|
|
@@ -80,7 +83,7 @@ class BaseBackend(ABC):
|
|
|
80
83
|
|
|
81
84
|
# Publisher
|
|
82
85
|
@abstractmethod
|
|
83
|
-
def publish(self, routing_key: str, message: "
|
|
86
|
+
def publish(self, routing_key: str, message: "Event") -> bool: ...
|
|
84
87
|
|
|
85
88
|
@abstractmethod
|
|
86
89
|
def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
|
|
@@ -7,7 +7,8 @@ from ._base import BaseBackend
|
|
|
7
7
|
logger = logging.getLogger(__name__)
|
|
8
8
|
|
|
9
9
|
if TYPE_CHECKING:
|
|
10
|
-
from ..
|
|
10
|
+
from ..event import Event
|
|
11
|
+
from ..types import UserCallback
|
|
11
12
|
|
|
12
13
|
|
|
13
14
|
class ConsoleBackend(BaseBackend):
|
|
@@ -15,7 +16,13 @@ class ConsoleBackend(BaseBackend):
|
|
|
15
16
|
super().__init__(url)
|
|
16
17
|
self.stream = self.get_option("stream", "stdout")
|
|
17
18
|
|
|
18
|
-
def
|
|
19
|
+
def configure_queue_routing(self) -> dict[str, list[str]]: # pragma: no cover
|
|
20
|
+
return {}
|
|
21
|
+
|
|
22
|
+
def configure_exchanges(self) -> None: # pragma: no cover
|
|
23
|
+
pass
|
|
24
|
+
|
|
25
|
+
def publish(self, routing_key: str, message: "Event") -> bool:
|
|
19
26
|
stream = getattr(sys, self.stream)
|
|
20
27
|
stream.write(f"routing_key:{routing_key} message:{message}\n")
|
|
21
28
|
return True
|
|
@@ -6,13 +6,20 @@ from ._base import BaseBackend
|
|
|
6
6
|
logger = logging.getLogger(__name__)
|
|
7
7
|
|
|
8
8
|
if TYPE_CHECKING:
|
|
9
|
-
from ..
|
|
9
|
+
from ..event import Event
|
|
10
|
+
from ..types import UserCallback
|
|
10
11
|
|
|
11
12
|
|
|
12
13
|
class DebugBackend(BaseBackend):
|
|
14
|
+
def configure_queue_routing(self) -> dict[str, list[str]]: # pragma: no cover
|
|
15
|
+
return {}
|
|
16
|
+
|
|
17
|
+
def configure_exchanges(self) -> None: # pragma: no cover
|
|
18
|
+
pass
|
|
19
|
+
|
|
13
20
|
def __init__(self, url: str) -> None:
|
|
14
21
|
super().__init__(url)
|
|
15
|
-
self.messages: list[tuple[str,
|
|
22
|
+
self.messages: list[tuple[str, Event]] = []
|
|
16
23
|
|
|
17
24
|
def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
|
|
18
25
|
pass
|
|
@@ -23,6 +30,6 @@ class DebugBackend(BaseBackend):
|
|
|
23
30
|
def disconnect(self) -> None:
|
|
24
31
|
pass
|
|
25
32
|
|
|
26
|
-
def publish(self, routing_key: str, message: "
|
|
33
|
+
def publish(self, routing_key: str, message: "Event") -> bool:
|
|
27
34
|
self.messages.append((routing_key, message))
|
|
28
35
|
return True
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import json
|
|
2
1
|
import logging
|
|
3
2
|
import socket
|
|
4
3
|
import time
|
|
4
|
+
from collections import defaultdict
|
|
5
|
+
from contextlib import suppress
|
|
5
6
|
from typing import TYPE_CHECKING
|
|
6
7
|
|
|
7
8
|
import pika.channel
|
|
@@ -13,20 +14,20 @@ from pika.exchange_type import ExchangeType
|
|
|
13
14
|
from streaming.config import CONFIG
|
|
14
15
|
from streaming.utils import exchange_exists
|
|
15
16
|
|
|
17
|
+
from ..event import Event
|
|
16
18
|
from ..exceptions import (
|
|
17
19
|
AuthorizationError,
|
|
18
|
-
StreamingCallbackError,
|
|
19
20
|
StreamingCallbackFailure,
|
|
21
|
+
StreamingCallbackRetryError,
|
|
20
22
|
StreamingConfigError,
|
|
21
23
|
)
|
|
22
|
-
from ..utils import json_dumps
|
|
23
24
|
from ._base import BaseBackend
|
|
24
25
|
|
|
25
26
|
if TYPE_CHECKING:
|
|
26
27
|
from pika.adapters.blocking_connection import BlockingChannel
|
|
27
28
|
from pika.spec import Basic, BasicProperties
|
|
28
29
|
|
|
29
|
-
from streaming.types import
|
|
30
|
+
from streaming.types import UserCallback
|
|
30
31
|
|
|
31
32
|
logger = logging.getLogger(__name__)
|
|
32
33
|
|
|
@@ -49,12 +50,11 @@ class Callback:
|
|
|
49
50
|
self.user_callback(self.queue_name, ch, method, properties, body)
|
|
50
51
|
if self.ack:
|
|
51
52
|
ch.basic_ack(delivery_tag=method.delivery_tag) # type: ignore[arg-type]
|
|
52
|
-
except
|
|
53
|
-
evt:
|
|
53
|
+
except StreamingCallbackRetryError as e:
|
|
54
|
+
evt: Event = Event.unmarshal(body)
|
|
55
|
+
logger.debug(f"StreamingCallbackError {evt.id}", exc_info=e)
|
|
54
56
|
retries = int(properties.headers.get("x-retries", 0)) # type: ignore[union-attr]
|
|
55
|
-
ch.basic_ack(method.delivery_tag) # type: ignore[arg-type]
|
|
56
57
|
self.backend._handle_retry(evt, ch, method, retries)
|
|
57
|
-
logger.debug("StreamingCallbackError", exc_info=e)
|
|
58
58
|
except StreamingCallbackFailure as e:
|
|
59
59
|
logger.error(f"Callback failure: {e}", exc_info=e)
|
|
60
60
|
except Exception as e:
|
|
@@ -89,19 +89,23 @@ class RabbitMQBackend(BaseBackend):
|
|
|
89
89
|
)
|
|
90
90
|
self.channel.exchange_declare(self.retry_exchange, exchange_type=ExchangeType.direct, durable=True)
|
|
91
91
|
|
|
92
|
-
def configure_queue_routing(self) ->
|
|
92
|
+
def configure_queue_routing(self) -> dict[str, list[str]]:
|
|
93
93
|
self.check_channel()
|
|
94
94
|
if not exchange_exists(self.channel, self.exchange):
|
|
95
95
|
raise StreamingConfigError("Exchange not found")
|
|
96
|
-
|
|
96
|
+
ret = defaultdict(list)
|
|
97
97
|
for alias, config in CONFIG.QUEUES.items():
|
|
98
98
|
real_name = self.get_real_queue_name(alias)
|
|
99
99
|
logger.debug(f"Declaring queue '{real_name}'")
|
|
100
100
|
self.channel.queue_declare(queue=real_name, durable=True, arguments=None) # type: ignore[union-attr]
|
|
101
|
-
binding_keys
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
101
|
+
if binding_keys := config.get("routing", []):
|
|
102
|
+
for binding_key in binding_keys:
|
|
103
|
+
logger.debug("Listening on queue '%s' routed by '%s'", alias, binding_key)
|
|
104
|
+
self.channel.queue_bind(exchange=self.exchange, queue=real_name, routing_key=binding_key) # type: ignore[union-attr]
|
|
105
|
+
ret[alias].append(binding_key)
|
|
106
|
+
else:
|
|
107
|
+
ret[alias].append("warning no routing defined")
|
|
108
|
+
return ret
|
|
105
109
|
|
|
106
110
|
def connect(self, raise_if_error: bool = False) -> None:
|
|
107
111
|
logger.debug("Connecting to %s:%s", self.host, self.port)
|
|
@@ -148,16 +152,15 @@ class RabbitMQBackend(BaseBackend):
|
|
|
148
152
|
def disconnect(self) -> None:
|
|
149
153
|
try:
|
|
150
154
|
if self._connection:
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
pass
|
|
155
|
+
with suppress(ConnectionClosedByBroker, AttributeError, ConnectionWrongStateError):
|
|
156
|
+
logger.debug("Closing RabbitMQ connection.")
|
|
157
|
+
self._connection.close()
|
|
155
158
|
finally:
|
|
156
159
|
self._connection = None
|
|
157
160
|
self.channel = None
|
|
158
161
|
|
|
159
162
|
# Publisher
|
|
160
|
-
def publish(self, routing_key: str, message: "
|
|
163
|
+
def publish(self, routing_key: str, message: "Event") -> bool:
|
|
161
164
|
try:
|
|
162
165
|
if not self.channel or self.channel.is_closed:
|
|
163
166
|
self.connect(True)
|
|
@@ -170,12 +173,14 @@ class RabbitMQBackend(BaseBackend):
|
|
|
170
173
|
|
|
171
174
|
# Listener
|
|
172
175
|
|
|
173
|
-
def _basic_publish(
|
|
176
|
+
def _basic_publish(
|
|
177
|
+
self, message: "Event", routing_key: str, retry_count: int = 0, exchange: str | None = None
|
|
178
|
+
) -> None:
|
|
174
179
|
self.check_channel()
|
|
175
180
|
self.channel.basic_publish( # type: ignore[union-attr]
|
|
176
|
-
exchange=self.exchange,
|
|
181
|
+
exchange=exchange or self.exchange,
|
|
177
182
|
routing_key=routing_key,
|
|
178
|
-
body=
|
|
183
|
+
body=message.marshall(),
|
|
179
184
|
properties=pika.BasicProperties(
|
|
180
185
|
delivery_mode=pika.spec.PERSISTENT_DELIVERY_MODE,
|
|
181
186
|
expiration=str(CONFIG.MESSAGE_TTL * 1000), # milliseconds
|
|
@@ -183,11 +188,15 @@ class RabbitMQBackend(BaseBackend):
|
|
|
183
188
|
),
|
|
184
189
|
)
|
|
185
190
|
|
|
186
|
-
def _handle_retry(self, message: "
|
|
191
|
+
def _handle_retry(self, message: "Event", ch: "BlockingChannel", method: "Basic.Deliver", retries: int) -> None:
|
|
187
192
|
ch.basic_ack(method.delivery_tag) # type: ignore[arg-type]
|
|
188
193
|
if retries < MAX_RETRIES:
|
|
189
194
|
delay = 2000 * (2**retries) # ms (exponential backoff)
|
|
190
|
-
delay_queue = f"{self.exchange}_retry_{delay}ms"
|
|
195
|
+
delay_queue = f"{self.exchange}_retry_{method.routing_key}_{delay}ms"
|
|
196
|
+
logger.debug(
|
|
197
|
+
f"Retrying message {message.id} ({retries + 1}/{MAX_RETRIES}) "
|
|
198
|
+
f"with routing key '{method.routing_key}' in {delay / 1000}s"
|
|
199
|
+
)
|
|
191
200
|
|
|
192
201
|
# Declare a delay queue with TTL
|
|
193
202
|
ch.queue_declare(
|
|
@@ -196,12 +205,13 @@ class RabbitMQBackend(BaseBackend):
|
|
|
196
205
|
arguments={
|
|
197
206
|
"x-dead-letter-exchange": self.exchange, # after delay → back to main
|
|
198
207
|
"x-message-ttl": delay,
|
|
208
|
+
"x-dead-letter-routing-key": method.routing_key,
|
|
199
209
|
},
|
|
200
210
|
)
|
|
201
|
-
ch.queue_bind(delay_queue, self.retry_exchange, routing_key=
|
|
202
|
-
self._basic_publish(message, routing_key=
|
|
211
|
+
ch.queue_bind(delay_queue, self.retry_exchange, routing_key=delay_queue)
|
|
212
|
+
self._basic_publish(message, routing_key=delay_queue, retry_count=retries + 1, exchange=self.retry_exchange)
|
|
203
213
|
else:
|
|
204
|
-
logger.error(f"Dropping after {MAX_RETRIES} retries")
|
|
214
|
+
logger.error(f"Dropping message {message.id} after {MAX_RETRIES} retries")
|
|
205
215
|
|
|
206
216
|
def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
|
|
207
217
|
if self.channel is None:
|
|
@@ -214,9 +224,6 @@ class RabbitMQBackend(BaseBackend):
|
|
|
214
224
|
return
|
|
215
225
|
queues_to_listen = queues or configured_queues.keys()
|
|
216
226
|
for queue_name in queues_to_listen:
|
|
217
|
-
if queue_name not in configured_queues:
|
|
218
|
-
logger.error(f"Queue '{queue_name}' not found in configured listening queues. Ignored.")
|
|
219
|
-
continue
|
|
220
227
|
real_queue_name = self.get_real_queue_name(queue_name)
|
|
221
228
|
_callback = Callback(queue_name, self, callback, ack=ack)
|
|
222
229
|
self.channel.basic_consume(queue=real_queue_name, on_message_callback=_callback, auto_ack=False) # type: ignore[union-attr]
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import uuid
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
from typing import TYPE_CHECKING, Any
|
|
5
|
+
|
|
6
|
+
from .utils import StreamingJSONEncoder
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from .types import JSON, EventType
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Event:
|
|
13
|
+
def __init__(
|
|
14
|
+
self,
|
|
15
|
+
*,
|
|
16
|
+
key: str,
|
|
17
|
+
payload: "JSON",
|
|
18
|
+
value_type: "EventType" = "absolute",
|
|
19
|
+
timestamp: datetime | None = None,
|
|
20
|
+
message_id: str | None = None,
|
|
21
|
+
) -> None:
|
|
22
|
+
self.timestamp = timestamp or datetime.now()
|
|
23
|
+
self.key = key
|
|
24
|
+
self.payload = payload
|
|
25
|
+
self.value_type = value_type
|
|
26
|
+
self.id = message_id or uuid.uuid4()
|
|
27
|
+
|
|
28
|
+
def marshall(self) -> bytes:
|
|
29
|
+
return json.dumps(
|
|
30
|
+
{
|
|
31
|
+
"id": self.id,
|
|
32
|
+
"timestamp": self.timestamp.isoformat(),
|
|
33
|
+
"key": self.key,
|
|
34
|
+
"payload": self.payload,
|
|
35
|
+
},
|
|
36
|
+
cls=StreamingJSONEncoder,
|
|
37
|
+
).encode()
|
|
38
|
+
|
|
39
|
+
@classmethod
|
|
40
|
+
def unmarshal(cls, body: bytes) -> "Event":
|
|
41
|
+
data = json.loads(body.decode())
|
|
42
|
+
return cls(
|
|
43
|
+
message_id=data["id"],
|
|
44
|
+
payload=data["payload"],
|
|
45
|
+
key=data["key"],
|
|
46
|
+
timestamp=datetime.fromisoformat(data["timestamp"]),
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
@classmethod
|
|
50
|
+
def build(cls, key: str, data: Any, value_type: "EventType") -> "Event":
|
|
51
|
+
if isinstance(data, str):
|
|
52
|
+
payload: JSON = {"message": data}
|
|
53
|
+
else:
|
|
54
|
+
payload = data
|
|
55
|
+
return cls(key=key, payload=payload, value_type=value_type)
|
|
56
|
+
|
|
57
|
+
def as_dict(self) -> dict[str, Any]:
|
|
58
|
+
return {
|
|
59
|
+
"id": self.id,
|
|
60
|
+
"timestamp": self.timestamp.isoformat(),
|
|
61
|
+
"key": self.key,
|
|
62
|
+
"payload": self.payload,
|
|
63
|
+
"value_type": self.value_type,
|
|
64
|
+
}
|
|
@@ -14,7 +14,7 @@ from .utils import make_event
|
|
|
14
14
|
|
|
15
15
|
if TYPE_CHECKING:
|
|
16
16
|
from .backends._base import BaseBackend
|
|
17
|
-
from .
|
|
17
|
+
from .event import Event
|
|
18
18
|
|
|
19
19
|
logger = logging.getLogger(__name__)
|
|
20
20
|
|
|
@@ -37,10 +37,10 @@ class ChangeManager:
|
|
|
37
37
|
for field in sender._meta.fields:
|
|
38
38
|
payload["fields"][field.name] = str(getattr(instance, field.name))
|
|
39
39
|
routing_key = f"{sender._meta.app_label}.{sender._meta.model_name}.save"
|
|
40
|
-
message:
|
|
40
|
+
message: Event = make_event(payload, key=routing_key)
|
|
41
41
|
self.notify(routing_key, message)
|
|
42
42
|
|
|
43
|
-
def notify(self, routing_key: str, event: "
|
|
43
|
+
def notify(self, routing_key: str, event: "Event") -> bool:
|
|
44
44
|
logger.debug("notifying [%s] %s", routing_key, event)
|
|
45
45
|
return self.backend.publish(routing_key, event)
|
|
46
46
|
|
|
@@ -74,7 +74,7 @@ class ThreadedChangeManager(ChangeManager):
|
|
|
74
74
|
except queue.Empty:
|
|
75
75
|
continue
|
|
76
76
|
|
|
77
|
-
def notify(self, routing_key: str, event: "
|
|
77
|
+
def notify(self, routing_key: str, event: "Event") -> bool:
|
|
78
78
|
self.queue.put((routing_key, event))
|
|
79
79
|
self.start()
|
|
80
80
|
return True
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
from collections.abc import Callable
|
|
2
|
-
from
|
|
3
|
-
from typing import Literal, TypedDict
|
|
2
|
+
from typing import Literal
|
|
4
3
|
from uuid import UUID
|
|
5
4
|
|
|
6
5
|
from pika.adapters.blocking_connection import BlockingChannel
|
|
@@ -9,11 +8,6 @@ from pika.spec import Basic, BasicProperties
|
|
|
9
8
|
JsonT = None | bool | int | float | str | list["JsonT"] | dict[str, "JsonT"] | UUID
|
|
10
9
|
JSON = dict[str, JsonT]
|
|
11
10
|
|
|
11
|
+
EventType = Literal["absolute", "delta", "event"]
|
|
12
12
|
PikaCallback = Callable[[BlockingChannel, Basic.Deliver, BasicProperties, bytes], None]
|
|
13
13
|
UserCallback = Callable[[str, BlockingChannel, Basic.Deliver, BasicProperties, bytes], None]
|
|
14
|
-
|
|
15
|
-
class EventType(TypedDict):
|
|
16
|
-
timestamp: str | datetime
|
|
17
|
-
event: str
|
|
18
|
-
payload: JSON
|
|
19
|
-
type: Literal["absolute", "delta", "event"]
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
import datetime
|
|
2
|
-
import json
|
|
3
1
|
import socket
|
|
4
2
|
from typing import TYPE_CHECKING, Any
|
|
5
3
|
|
|
@@ -9,7 +7,9 @@ from pika.adapters.blocking_connection import BlockingChannel
|
|
|
9
7
|
from pika.exceptions import ChannelClosedByBroker
|
|
10
8
|
|
|
11
9
|
if TYPE_CHECKING:
|
|
12
|
-
from streaming.
|
|
10
|
+
from streaming.event import Event
|
|
11
|
+
from streaming.types import JSON
|
|
12
|
+
|
|
13
13
|
|
|
14
14
|
MINUTE = 60
|
|
15
15
|
HOUR = MINUTE * 60
|
|
@@ -18,49 +18,21 @@ DAY = HOUR * 24
|
|
|
18
18
|
|
|
19
19
|
class StreamingJSONEncoder(DjangoJSONEncoder):
|
|
20
20
|
def default(self, o: Any) -> Any:
|
|
21
|
-
if isinstance(o, datetime.datetime):
|
|
22
|
-
return o.isoformat()
|
|
23
21
|
if isinstance(o, models.Model):
|
|
24
22
|
return str(o)
|
|
25
23
|
return super().default(o)
|
|
26
24
|
|
|
27
25
|
|
|
28
|
-
def json_dumps(obj: Any) -> str:
|
|
29
|
-
return json.dumps(obj, cls=StreamingJSONEncoder)
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
def json_loads(obj: str) -> "JSON":
|
|
33
|
-
def try_parse_iso(value: Any) -> Any:
|
|
34
|
-
if isinstance(value, str):
|
|
35
|
-
try:
|
|
36
|
-
return datetime.datetime.fromisoformat(value)
|
|
37
|
-
except ValueError:
|
|
38
|
-
return value
|
|
39
|
-
return value
|
|
40
|
-
|
|
41
|
-
def hook(dct: "JSON") -> Any:
|
|
42
|
-
return {k: try_parse_iso(v) for k, v in dct.items()}
|
|
43
|
-
|
|
44
|
-
return json.loads(obj, object_hook=hook) # type: ignore[no-any-return]
|
|
45
|
-
|
|
46
|
-
|
|
47
26
|
def parse_bool(value: Any) -> bool:
|
|
48
27
|
if isinstance(value, str):
|
|
49
28
|
return value.lower() in ("yes", "true", "1", "y", "t")
|
|
50
29
|
return value in [1, True]
|
|
51
30
|
|
|
52
31
|
|
|
53
|
-
def make_event(message: "str | JSON", *,
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
payload = message
|
|
58
|
-
return {
|
|
59
|
-
"event": event,
|
|
60
|
-
"type": "absolute",
|
|
61
|
-
"timestamp": datetime.datetime.now(),
|
|
62
|
-
"payload": payload,
|
|
63
|
-
}
|
|
32
|
+
def make_event(message: "str | JSON", *, key: str = "") -> "Event":
|
|
33
|
+
from streaming.event import Event
|
|
34
|
+
|
|
35
|
+
return Event.build(key=key, data=message, value_type="absolute")
|
|
64
36
|
|
|
65
37
|
|
|
66
38
|
def get_local_ip() -> str:
|
|
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
|
|
|
28
28
|
commit_id: COMMIT_ID
|
|
29
29
|
__commit_id__: COMMIT_ID
|
|
30
30
|
|
|
31
|
-
__version__ = version = '0.
|
|
32
|
-
__version_tuple__ = version_tuple = (0,
|
|
31
|
+
__version__ = version = '0.6.0'
|
|
32
|
+
__version_tuple__ = version_tuple = (0, 6, 0)
|
|
33
33
|
|
|
34
34
|
__commit_id__ = commit_id = None
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|