django-streaming 0.5__tar.gz → 0.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. {django_streaming-0.5 → django_streaming-0.7}/.gitignore +1 -0
  2. {django_streaming-0.5 → django_streaming-0.7}/PKG-INFO +1 -1
  3. {django_streaming-0.5 → django_streaming-0.7}/src/streaming/__cli__.py +71 -69
  4. {django_streaming-0.5 → django_streaming-0.7}/src/streaming/backends/_base.py +9 -6
  5. {django_streaming-0.5 → django_streaming-0.7}/src/streaming/backends/console.py +9 -2
  6. {django_streaming-0.5 → django_streaming-0.7}/src/streaming/backends/debug.py +10 -3
  7. {django_streaming-0.5 → django_streaming-0.7}/src/streaming/backends/rabbitmq.py +48 -41
  8. django_streaming-0.7/src/streaming/callbacks.py +22 -0
  9. {django_streaming-0.5 → django_streaming-0.7}/src/streaming/config.py +1 -0
  10. django_streaming-0.7/src/streaming/event.py +64 -0
  11. django_streaming-0.7/src/streaming/exceptions.py +35 -0
  12. {django_streaming-0.5 → django_streaming-0.7}/src/streaming/manager.py +4 -4
  13. django_streaming-0.5/src/streaming/types.pyi → django_streaming-0.7/src/streaming/types.py +3 -9
  14. django_streaming-0.7/src/streaming/utils.py +111 -0
  15. {django_streaming-0.5 → django_streaming-0.7}/src/streaming/version.py +2 -2
  16. django_streaming-0.5/src/streaming/api.py +0 -0
  17. django_streaming-0.5/src/streaming/exceptions.py +0 -27
  18. django_streaming-0.5/src/streaming/utils.py +0 -78
  19. {django_streaming-0.5 → django_streaming-0.7}/LICENSE +0 -0
  20. {django_streaming-0.5 → django_streaming-0.7}/README.md +0 -0
  21. {django_streaming-0.5 → django_streaming-0.7}/pyproject.toml +0 -0
  22. {django_streaming-0.5 → django_streaming-0.7}/src/streaming/__init__.py +0 -0
  23. {django_streaming-0.5 → django_streaming-0.7}/src/streaming/apps.py +0 -0
  24. {django_streaming-0.5 → django_streaming-0.7}/src/streaming/backends/__init__.py +0 -0
  25. {django_streaming-0.5 → django_streaming-0.7}/src/streaming/py.typed +0 -0
@@ -16,5 +16,6 @@ coverage.xml
16
16
  src/streaming/version.py
17
17
  !.gitignore
18
18
  !.github
19
+ !docs/**/.pages
19
20
  !tests/.coveragerc
20
21
  !.python-version
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: django-streaming
3
- Version: 0.5
3
+ Version: 0.7
4
4
  Summary: A project for streaming data.
5
5
  Project-URL: Documentation, https://github.com/UNICEF/django-streaming
6
6
  Project-URL: Homepage, https://github.com/UNICEF/django-streaming
@@ -1,38 +1,34 @@
1
1
  import json
2
2
  import logging
3
+ from collections.abc import Iterable
3
4
  from typing import TYPE_CHECKING
4
5
 
5
6
  import click
6
7
  from click import ClickException
7
8
  from colorama import Fore, Style
8
9
  from django.core.exceptions import ImproperlyConfigured
9
- from pika.adapters.blocking_connection import BlockingChannel
10
+ from django.utils.module_loading import import_string
10
11
  from pika.exceptions import ChannelClosedByBroker
11
- from pika.spec import Basic, BasicProperties
12
12
 
13
13
  from .backends import RabbitMQBackend, get_backend
14
- from .exceptions import AuthorizationError, StreamingConfigError
15
- from .utils import make_event
14
+ from .config import CONFIG
15
+ from .exceptions import AuthorizationError, CallbackRetry, StreamingConfigError
16
+ from .utils import LevelFormatter, check_callback, make_event
16
17
 
17
18
  if TYPE_CHECKING:
18
- from .types import JSON, EventType
19
+ from .event import Event
19
20
 
20
21
 
21
22
  logger = logging.getLogger(__name__)
23
+ INFO_LINE = f"{Fore.YELLOW}%-16s: {Style.RESET_ALL}%s"
22
24
 
23
25
 
24
26
  def _dump_info(backend: "RabbitMQBackend") -> None:
25
- from streaming.config import CONFIG
26
-
27
- line = f"{Fore.YELLOW}%-16s: {Style.RESET_ALL}%s"
28
- click.secho(line % ("Server", f"{backend.host}:{backend.port}"))
29
- click.secho(line % ("VirtualHost", f"{backend.virtual_host}"))
30
- click.secho(line % ("Exchange", f"{backend.exchange}"))
31
- click.secho(line % ("Queues", ""))
32
- for alias, config in CONFIG.QUEUES.items():
33
- click.secho(line % (f" {alias}", f"{config}"))
34
- click.secho(line % ("Timeout", f"{backend.timeout}"))
35
- click.secho(line % ("Client Name", f"{backend.client_name}"))
27
+ click.secho(INFO_LINE % ("Server", f"{backend.host}:{backend.port}"))
28
+ click.secho(INFO_LINE % ("VirtualHost", f"{backend.virtual_host}"))
29
+ click.secho(INFO_LINE % ("Exchange", f"{backend.exchange}"))
30
+ click.secho(INFO_LINE % ("Timeout", f"{backend.timeout}"))
31
+ click.secho(INFO_LINE % ("Client Name", f"{backend.client_name}"))
36
32
 
37
33
 
38
34
  def assert_backend() -> "RabbitMQBackend":
@@ -43,6 +39,19 @@ def assert_backend() -> "RabbitMQBackend":
43
39
  return backend
44
40
 
45
41
 
42
+ def configure_logging(debug: bool, loggers: Iterable[str] = ("streaming",)) -> None:
43
+ # This should be called only from a click command
44
+ formatter = LevelFormatter()
45
+ ch = logging.StreamHandler()
46
+ ch.setFormatter(formatter)
47
+ for log_name in loggers:
48
+ logr = logging.getLogger(log_name)
49
+ logr.handlers = []
50
+ if debug:
51
+ logr.setLevel(logging.DEBUG)
52
+ logr.addHandler(ch)
53
+
54
+
46
55
  @click.group()
47
56
  def cli() -> None:
48
57
  """Streaming CLI."""
@@ -57,18 +66,24 @@ def cli() -> None:
57
66
 
58
67
 
59
68
  @cli.command()
60
- @click.option("--client-name", default=None, help="Override client name")
61
- def configure(client_name: str) -> None:
69
+ @click.option("--queues/--no-queues", "queues", is_flag=True, default=False, help="Debug mode")
70
+ @click.option("--debug", is_flag=True, help="Debug mode")
71
+ def configure(queues: bool = False, debug: bool = False) -> None:
62
72
  from streaming.config import CONFIG
63
73
 
74
+ routing = None
75
+ configure_logging(debug)
64
76
  backend: RabbitMQBackend = assert_backend()
65
- if client_name:
66
- backend.client_name = client_name
67
77
  try:
68
78
  backend.connect(True)
69
79
  backend.configure_exchanges()
70
- backend.configure_queue_routing()
80
+ if queues:
81
+ routing = backend.configure_queue_routing()
71
82
  check.callback() # type: ignore[misc]
83
+ if routing:
84
+ click.secho(INFO_LINE % ("Queues", ""))
85
+ for k, v in routing.items():
86
+ click.secho(INFO_LINE % (f" {k}", "; ".join(v)))
72
87
  except AuthorizationError as e:
73
88
  click.secho(f"Unable to connect using {CONFIG.BROKER_URL}", fg="red", err=True)
74
89
  raise ClickException(str(e)) from e
@@ -84,12 +99,7 @@ def configure(client_name: str) -> None:
84
99
  @click.option("--debug", is_flag=True, help="Debug mode")
85
100
  def send(routing_key: str, message: str, client_name: str, debug: bool) -> None:
86
101
  backend = assert_backend()
87
- logger = logging.getLogger("streaming")
88
- if debug:
89
- logger.setLevel(logging.DEBUG)
90
- logger.addHandler(logging.StreamHandler())
91
- else:
92
- logger.handlers = []
102
+ configure_logging(debug)
93
103
 
94
104
  if client_name:
95
105
  backend.client_name = client_name
@@ -99,63 +109,55 @@ def send(routing_key: str, message: str, client_name: str, debug: bool) -> None:
99
109
  payload = {
100
110
  "message": message,
101
111
  }
102
- msg: EventType = make_event(payload, event="Test")
112
+ msg: Event = make_event(payload, key="Test")
103
113
  backend.publish(routing_key, msg)
104
114
  click.secho(f"Sent: {msg}")
105
115
  backend.disconnect()
106
116
 
107
117
 
108
- def _listen(queues: list[str], payload: bool, pretty: bool, client_name: str) -> None:
109
- backend = assert_backend()
118
+ @cli.command()
119
+ @click.option("-q", "--queues", multiple=True, help="Queue name to listen to")
120
+ @click.option("-cb", "--callback", default=None, help="User callback")
121
+ @click.option("--debug", is_flag=True, help="Debug mode")
122
+ def listen( # noqa PLR0913
123
+ queues: list[str], callback: str | None = None, debug: bool = False, dry_run: bool = False
124
+ ) -> None:
125
+ """Listens for streaming events."""
126
+ configure_logging(debug)
110
127
 
111
- if client_name:
112
- backend.client_name = client_name
128
+ if not callback:
129
+ callback = CONFIG.LISTEN_CALLBACK
113
130
 
114
- def callback(
115
- queue_name: str, ch: BlockingChannel, method: Basic.Deliver, properties: BasicProperties, body: bytes
116
- ) -> None:
117
- message: EventType = json.loads(body.decode())
118
- click.echo(
119
- f"{Fore.GREEN}{message['timestamp']} "
120
- f"[{queue_name}]"
121
- f"{Fore.LIGHTWHITE_EX} [{message['type']}]"
122
- f"{message['event']} "
123
- )
124
- extra: str | JSON
125
- if payload:
126
- if pretty:
127
- extra = json.dumps(message["payload"], indent=4)
128
- else:
129
- extra = message["payload"]
130
- click.echo(f"{Fore.YELLOW}{extra}{Fore.RESET}")
131
+ try:
132
+ cb = import_string(callback)
133
+ if not check_callback(cb):
134
+ raise ClickException(f"Callback {callback} is not a valid callback")
135
+ except ImportError:
136
+ raise ClickException(f"Callback {callback} is not a valid callback") from None
131
137
 
138
+ backend = assert_backend()
132
139
  try:
133
- backend.connect()
134
140
  _dump_info(backend)
135
- backend.listen(callback, queues=queues)
136
- except KeyboardInterrupt:
141
+ backend.connect(True)
142
+ backend.listen(cb, queues=queues, ack=not dry_run)
143
+ except ChannelClosedByBroker as e:
144
+ logger.debug(e)
145
+ click.secho(str(e), fg="red")
146
+ if "no queue" in str(e):
147
+ click.secho("Did you run 'stream configure --queues'", fg="red")
148
+ except CallbackRetry as e:
149
+ logger.debug(e)
150
+ except (StreamingConfigError, ImportError) as e:
151
+ logger.debug(e)
152
+ click.secho(str(e), fg="red")
153
+ click.get_current_context().exit(2)
154
+ except KeyboardInterrupt as e:
155
+ logger.debug(e)
137
156
  click.secho("Stopping listener.", fg="yellow")
138
157
  finally:
139
158
  backend.disconnect()
140
159
 
141
160
 
142
- @cli.command()
143
- @click.option("-q", "--queues", multiple=True, help="Queue name to listen to")
144
- @click.option("-c", "--client-name", default=None, help="Override client name")
145
- @click.option("--payload", default=False, is_flag=True, help="Print payload")
146
- @click.option("--autoreload", "reload", is_flag=True, help="Enable auto-reloading.")
147
- @click.option("--pretty", is_flag=True, help="Pretty-print payload.")
148
- def listen(queues: list[str], payload: bool, reload: bool, pretty: bool, client_name: str) -> None:
149
- """Listens for streaming events."""
150
- if reload:
151
- from django.utils import autoreload
152
-
153
- click.secho("Starting listener with autoreload...", fg="yellow")
154
- autoreload.run_with_reloader(_listen, queues=queues, payload=payload, pretty=pretty, client_name=client_name)
155
- else:
156
- _listen(queues=queues, payload=payload, pretty=pretty, client_name=client_name)
157
-
158
-
159
161
  @cli.command()
160
162
  def purge() -> None:
161
163
  """Purges all messages from the configured queues."""
@@ -14,7 +14,8 @@ from ..exceptions import (
14
14
  if TYPE_CHECKING:
15
15
  from pika.adapters.blocking_connection import BlockingChannel, BlockingConnection
16
16
 
17
- from streaming.types import EventType, UserCallback
17
+ from ..event import Event
18
+ from ..types import UserCallback
18
19
 
19
20
 
20
21
  logger = logging.getLogger(__name__)
@@ -41,7 +42,7 @@ class BaseBackend(ABC):
41
42
  self.timeout = float(self.get_option("timeout", 0.5))
42
43
  self.virtual_host = self.get_option("vhost", "/")
43
44
  # listener
44
- self.client_name = CONFIG.CLIENT_NAME or get_local_ip()
45
+ self.client_name = CONFIG.CLIENT_NAME or self.get_option("client_name", get_local_ip())
45
46
  atexit.register(self.disconnect)
46
47
 
47
48
  def get_option(self, name: str, default: Any = "") -> Any:
@@ -55,11 +56,13 @@ class BaseBackend(ABC):
55
56
  def client_name(self, name: str) -> None:
56
57
  self.__client_name = name
57
58
 
59
+ @abstractmethod
58
60
  def configure_exchanges(self) -> None: # noqa: B027
59
- pass
61
+ ...
60
62
 
61
- def configure_queue_routing(self) -> None: # noqa: B027
62
- pass
63
+ @abstractmethod
64
+ def configure_queue_routing(self) -> dict[str, list[str]]: # noqa: B027
65
+ ...
63
66
 
64
67
  def get_real_queue_name(self, name: str) -> str:
65
68
  return f"{self.client_name}:{name}"
@@ -80,7 +83,7 @@ class BaseBackend(ABC):
80
83
 
81
84
  # Publisher
82
85
  @abstractmethod
83
- def publish(self, routing_key: str, message: "EventType") -> bool: ...
86
+ def publish(self, routing_key: str, message: "Event") -> bool: ...
84
87
 
85
88
  @abstractmethod
86
89
  def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
@@ -7,7 +7,8 @@ from ._base import BaseBackend
7
7
  logger = logging.getLogger(__name__)
8
8
 
9
9
  if TYPE_CHECKING:
10
- from ..types import EventType, UserCallback
10
+ from ..event import Event
11
+ from ..types import UserCallback
11
12
 
12
13
 
13
14
  class ConsoleBackend(BaseBackend):
@@ -15,7 +16,13 @@ class ConsoleBackend(BaseBackend):
15
16
  super().__init__(url)
16
17
  self.stream = self.get_option("stream", "stdout")
17
18
 
18
- def publish(self, routing_key: str, message: "EventType") -> bool:
19
+ def configure_queue_routing(self) -> dict[str, list[str]]: # pragma: no cover
20
+ return {}
21
+
22
+ def configure_exchanges(self) -> None: # pragma: no cover
23
+ pass
24
+
25
+ def publish(self, routing_key: str, message: "Event") -> bool:
19
26
  stream = getattr(sys, self.stream)
20
27
  stream.write(f"routing_key:{routing_key} message:{message}\n")
21
28
  return True
@@ -6,13 +6,20 @@ from ._base import BaseBackend
6
6
  logger = logging.getLogger(__name__)
7
7
 
8
8
  if TYPE_CHECKING:
9
- from ..types import EventType, UserCallback
9
+ from ..event import Event
10
+ from ..types import UserCallback
10
11
 
11
12
 
12
13
  class DebugBackend(BaseBackend):
14
+ def configure_queue_routing(self) -> dict[str, list[str]]: # pragma: no cover
15
+ return {}
16
+
17
+ def configure_exchanges(self) -> None: # pragma: no cover
18
+ pass
19
+
13
20
  def __init__(self, url: str) -> None:
14
21
  super().__init__(url)
15
- self.messages: list[tuple[str, EventType]] = []
22
+ self.messages: list[tuple[str, Event]] = []
16
23
 
17
24
  def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
18
25
  pass
@@ -23,6 +30,6 @@ class DebugBackend(BaseBackend):
23
30
  def disconnect(self) -> None:
24
31
  pass
25
32
 
26
- def publish(self, routing_key: str, message: "EventType") -> bool:
33
+ def publish(self, routing_key: str, message: "Event") -> bool:
27
34
  self.messages.append((routing_key, message))
28
35
  return True
@@ -1,32 +1,34 @@
1
- import json
2
1
  import logging
3
2
  import socket
4
3
  import time
4
+ from collections import defaultdict
5
+ from contextlib import suppress
5
6
  from typing import TYPE_CHECKING
6
7
 
7
8
  import pika.channel
8
9
  import pika.exceptions
9
10
  from pika import PlainCredentials
10
- from pika.exceptions import ConnectionClosedByBroker, ConnectionWrongStateError
11
+ from pika.exceptions import AMQPConnectionError, ConnectionClosed
11
12
  from pika.exchange_type import ExchangeType
12
13
 
13
14
  from streaming.config import CONFIG
14
15
  from streaming.utils import exchange_exists
15
16
 
17
+ from ..event import Event
16
18
  from ..exceptions import (
17
19
  AuthorizationError,
18
- StreamingCallbackError,
19
- StreamingCallbackFailure,
20
+ CallbackError,
21
+ CallbackRetry,
22
+ CallbackSkipAck,
20
23
  StreamingConfigError,
21
24
  )
22
- from ..utils import json_dumps
23
25
  from ._base import BaseBackend
24
26
 
25
27
  if TYPE_CHECKING:
26
28
  from pika.adapters.blocking_connection import BlockingChannel
27
29
  from pika.spec import Basic, BasicProperties
28
30
 
29
- from streaming.types import EventType, UserCallback
31
+ from streaming.types import UserCallback
30
32
 
31
33
  logger = logging.getLogger(__name__)
32
34
 
@@ -34,12 +36,9 @@ MAX_RETRIES = 5
34
36
 
35
37
 
36
38
  class Callback:
37
- def __init__(
38
- self, queue_name: str, backend: "RabbitMQBackend", user_callback: "UserCallback", ack: bool = True
39
- ) -> None:
39
+ def __init__(self, queue_name: str, backend: "RabbitMQBackend", user_callback: "UserCallback") -> None:
40
40
  self.backend = backend
41
41
  self.user_callback: UserCallback = user_callback
42
- self.ack = ack
43
42
  self.queue_name = queue_name
44
43
 
45
44
  def __call__(
@@ -47,15 +46,14 @@ class Callback:
47
46
  ) -> None:
48
47
  try:
49
48
  self.user_callback(self.queue_name, ch, method, properties, body)
50
- if self.ack:
51
- ch.basic_ack(delivery_tag=method.delivery_tag) # type: ignore[arg-type]
52
- except StreamingCallbackError as e:
53
- evt: EventType = json.loads(body.decode())
49
+ ch.basic_ack(delivery_tag=method.delivery_tag) # type: ignore[arg-type]
50
+ except CallbackSkipAck as e:
51
+ logger.error(f"StreamingCallbackSkipAckException: {e}", exc_info=e)
52
+ except CallbackRetry:
53
+ evt: Event = Event.unmarshal(body)
54
54
  retries = int(properties.headers.get("x-retries", 0)) # type: ignore[union-attr]
55
- ch.basic_ack(method.delivery_tag) # type: ignore[arg-type]
56
55
  self.backend._handle_retry(evt, ch, method, retries)
57
- logger.debug("StreamingCallbackError", exc_info=e)
58
- except StreamingCallbackFailure as e:
56
+ except CallbackError as e:
59
57
  logger.error(f"Callback failure: {e}", exc_info=e)
60
58
  except Exception as e:
61
59
  logger.exception(f"Unexpected exception occurred: {e}", exc_info=e)
@@ -89,19 +87,26 @@ class RabbitMQBackend(BaseBackend):
89
87
  )
90
88
  self.channel.exchange_declare(self.retry_exchange, exchange_type=ExchangeType.direct, durable=True)
91
89
 
92
- def configure_queue_routing(self) -> None:
90
+ def configure_queue_routing(self) -> dict[str, list[str]]:
93
91
  self.check_channel()
94
92
  if not exchange_exists(self.channel, self.exchange):
95
93
  raise StreamingConfigError("Exchange not found")
96
-
94
+ ret = defaultdict(list)
97
95
  for alias, config in CONFIG.QUEUES.items():
98
96
  real_name = self.get_real_queue_name(alias)
99
97
  logger.debug(f"Declaring queue '{real_name}'")
100
98
  self.channel.queue_declare(queue=real_name, durable=True, arguments=None) # type: ignore[union-attr]
101
- binding_keys = config.get("routing", [])
102
- for binding_key in binding_keys:
103
- logger.debug("Listening on queue '%s' routed by '%s'", alias, binding_key)
104
- self.channel.queue_bind(exchange=self.exchange, queue=real_name, routing_key=binding_key) # type: ignore[union-attr]
99
+ if binding_keys := config.get("binding_keys", []):
100
+ options = config.get("options", {})
101
+ for binding_key in binding_keys:
102
+ logger.debug("Listening on queue '%s' routed by '%s'", alias, binding_key)
103
+ self.channel.queue_bind( # type: ignore[union-attr]
104
+ exchange=self.exchange, queue=real_name, routing_key=binding_key, arguments=options
105
+ )
106
+ ret[alias].append(binding_key)
107
+ else:
108
+ ret[alias].append("warning no routing defined")
109
+ return ret
105
110
 
106
111
  def connect(self, raise_if_error: bool = False) -> None:
107
112
  logger.debug("Connecting to %s:%s", self.host, self.port)
@@ -148,16 +153,15 @@ class RabbitMQBackend(BaseBackend):
148
153
  def disconnect(self) -> None:
149
154
  try:
150
155
  if self._connection:
151
- logger.debug("Closing RabbitMQ connection.")
152
- self._connection.close()
153
- except (ConnectionClosedByBroker, AttributeError, ConnectionWrongStateError):
154
- pass
156
+ with suppress(ConnectionClosed, AttributeError, AMQPConnectionError):
157
+ logger.debug("Closing RabbitMQ connection.")
158
+ self._connection.close()
155
159
  finally:
156
160
  self._connection = None
157
161
  self.channel = None
158
162
 
159
163
  # Publisher
160
- def publish(self, routing_key: str, message: "EventType") -> bool:
164
+ def publish(self, routing_key: str, message: "Event") -> bool:
161
165
  try:
162
166
  if not self.channel or self.channel.is_closed:
163
167
  self.connect(True)
@@ -170,12 +174,14 @@ class RabbitMQBackend(BaseBackend):
170
174
 
171
175
  # Listener
172
176
 
173
- def _basic_publish(self, message: "EventType", routing_key: str, retry_count: int = 0) -> None:
177
+ def _basic_publish(
178
+ self, message: "Event", routing_key: str, retry_count: int = 0, exchange: str | None = None
179
+ ) -> None:
174
180
  self.check_channel()
175
181
  self.channel.basic_publish( # type: ignore[union-attr]
176
- exchange=self.exchange,
182
+ exchange=exchange or self.exchange,
177
183
  routing_key=routing_key,
178
- body=json_dumps(message).encode(),
184
+ body=message.marshall(),
179
185
  properties=pika.BasicProperties(
180
186
  delivery_mode=pika.spec.PERSISTENT_DELIVERY_MODE,
181
187
  expiration=str(CONFIG.MESSAGE_TTL * 1000), # milliseconds
@@ -183,11 +189,15 @@ class RabbitMQBackend(BaseBackend):
183
189
  ),
184
190
  )
185
191
 
186
- def _handle_retry(self, message: "EventType", ch: "BlockingChannel", method: "Basic.Deliver", retries: int) -> None:
192
+ def _handle_retry(self, message: "Event", ch: "BlockingChannel", method: "Basic.Deliver", retries: int) -> None:
187
193
  ch.basic_ack(method.delivery_tag) # type: ignore[arg-type]
188
194
  if retries < MAX_RETRIES:
189
195
  delay = 2000 * (2**retries) # ms (exponential backoff)
190
- delay_queue = f"{self.exchange}_retry_{delay}ms"
196
+ delay_queue = f"{self.exchange}_retry_{method.routing_key}_{delay}ms"
197
+ logger.debug(
198
+ f"Retrying message {message.id} ({retries + 1}/{MAX_RETRIES}) "
199
+ f"with routing key '{method.routing_key}' in {delay / 1000}s"
200
+ )
191
201
 
192
202
  # Declare a delay queue with TTL
193
203
  ch.queue_declare(
@@ -196,29 +206,26 @@ class RabbitMQBackend(BaseBackend):
196
206
  arguments={
197
207
  "x-dead-letter-exchange": self.exchange, # after delay → back to main
198
208
  "x-message-ttl": delay,
209
+ "x-dead-letter-routing-key": method.routing_key,
199
210
  },
200
211
  )
201
- ch.queue_bind(delay_queue, self.retry_exchange, routing_key="task")
202
- self._basic_publish(message, routing_key=method.routing_key or "*", retry_count=retries + 1)
212
+ ch.queue_bind(delay_queue, self.retry_exchange, routing_key=delay_queue)
213
+ self._basic_publish(message, routing_key=delay_queue, retry_count=retries + 1, exchange=self.retry_exchange)
203
214
  else:
204
- logger.error(f"Dropping after {MAX_RETRIES} retries")
215
+ logger.error(f"Dropping message {message.id} after {MAX_RETRIES} retries")
205
216
 
206
217
  def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
207
218
  if self.channel is None:
208
219
  self.connect()
209
220
  self.check_channel()
210
-
211
221
  configured_queues = CONFIG.QUEUES
212
222
  if not configured_queues:
213
223
  logger.warning("No queues configured in settings.STREAMING['QUEUES']")
214
224
  return
215
225
  queues_to_listen = queues or configured_queues.keys()
216
226
  for queue_name in queues_to_listen:
217
- if queue_name not in configured_queues:
218
- logger.error(f"Queue '{queue_name}' not found in configured listening queues. Ignored.")
219
- continue
220
227
  real_queue_name = self.get_real_queue_name(queue_name)
221
- _callback = Callback(queue_name, self, callback, ack=ack)
228
+ _callback = Callback(queue_name, self, callback)
222
229
  self.channel.basic_consume(queue=real_queue_name, on_message_callback=_callback, auto_ack=False) # type: ignore[union-attr]
223
230
 
224
231
  logger.info("Waiting for messages. To exit press CTRL+C")
@@ -0,0 +1,22 @@
1
+ import logging
2
+ import sys
3
+
4
+ from colorama import Fore
5
+ from pika.adapters.blocking_connection import BlockingChannel
6
+ from pika.spec import Basic, BasicProperties
7
+
8
+ from streaming.event import Event
9
+
10
+ logger = logging.getLogger(__name__)
11
+
12
+
13
+ def default_callback(
14
+ queue_name: str, ch: BlockingChannel, method: Basic.Deliver, properties: BasicProperties, body: bytes
15
+ ) -> bool:
16
+ logger.debug("Invoking default callback")
17
+ message: Event = Event.unmarshal(body)
18
+ ack = f"{Fore.GREEN}ACK{Fore.RESET}"
19
+ sys.stdout.write(
20
+ f"{Fore.GREEN}{message.timestamp} [{queue_name}]{Fore.LIGHTWHITE_EX} [{message.key}] {message.id} {ack}\n"
21
+ )
22
+ return True
@@ -18,6 +18,7 @@ class StreamingConfig:
18
18
  "RETRY_DELAY": 1,
19
19
  "MESSAGE_TTL": 60 * 60 * 24 * 2, # 2 days
20
20
  "MANAGER_CLASS": "streaming.manager.ChangeManager",
21
+ "LISTEN_CALLBACK": "streaming.callbacks.default_callback",
21
22
  }
22
23
 
23
24
  def __init__(self) -> None:
@@ -0,0 +1,64 @@
1
+ import json
2
+ import uuid
3
+ from datetime import datetime
4
+ from typing import TYPE_CHECKING, Any
5
+
6
+ from .utils import StreamingJSONEncoder
7
+
8
+ if TYPE_CHECKING:
9
+ from .types import JSON, EventType
10
+
11
+
12
+ class Event:
13
+ def __init__(
14
+ self,
15
+ *,
16
+ key: str,
17
+ payload: "JSON",
18
+ value_type: "EventType" = "absolute",
19
+ timestamp: datetime | None = None,
20
+ message_id: str | None = None,
21
+ ) -> None:
22
+ self.timestamp = timestamp or datetime.now()
23
+ self.key = key
24
+ self.payload = payload
25
+ self.value_type = value_type
26
+ self.id = message_id or uuid.uuid4()
27
+
28
+ def marshall(self) -> bytes:
29
+ return json.dumps(
30
+ {
31
+ "id": self.id,
32
+ "timestamp": self.timestamp.isoformat(),
33
+ "key": self.key,
34
+ "payload": self.payload,
35
+ },
36
+ cls=StreamingJSONEncoder,
37
+ ).encode()
38
+
39
+ @classmethod
40
+ def unmarshal(cls, body: bytes) -> "Event":
41
+ data = json.loads(body.decode())
42
+ return cls(
43
+ message_id=data["id"],
44
+ payload=data["payload"],
45
+ key=data["key"],
46
+ timestamp=datetime.fromisoformat(data["timestamp"]),
47
+ )
48
+
49
+ @classmethod
50
+ def build(cls, key: str, data: Any, value_type: "EventType") -> "Event":
51
+ if isinstance(data, str):
52
+ payload: JSON = {"message": data}
53
+ else:
54
+ payload = data
55
+ return cls(key=key, payload=payload, value_type=value_type)
56
+
57
+ def as_dict(self) -> dict[str, Any]:
58
+ return {
59
+ "id": self.id,
60
+ "timestamp": self.timestamp.isoformat(),
61
+ "key": self.key,
62
+ "payload": self.payload,
63
+ "value_type": self.value_type,
64
+ }
@@ -0,0 +1,35 @@
1
+ import logging
2
+
3
+ logger = logging.getLogger(__name__)
4
+
5
+
6
+ class StreamingError(Exception):
7
+ pass
8
+
9
+
10
+ class StreamingException(Exception): # noqa N818
11
+ pass
12
+
13
+
14
+ class StreamingConfigError(StreamingError):
15
+ pass
16
+
17
+
18
+ class AuthorizationError(StreamingConfigError):
19
+ pass
20
+
21
+
22
+ class StreamingBackendError(StreamingError):
23
+ pass
24
+
25
+
26
+ class CallbackSkipAck(StreamingException): # noqa: N818
27
+ """Consume exception but do not acknowledge the message."""
28
+
29
+
30
+ class CallbackRetry(StreamingException): # noqa: N818
31
+ """Message will be re-queued."""
32
+
33
+
34
+ class CallbackError(StreamingError): # noqa: N818
35
+ """Message will be discarded."""
@@ -14,7 +14,7 @@ from .utils import make_event
14
14
 
15
15
  if TYPE_CHECKING:
16
16
  from .backends._base import BaseBackend
17
- from .types import EventType
17
+ from .event import Event
18
18
 
19
19
  logger = logging.getLogger(__name__)
20
20
 
@@ -37,10 +37,10 @@ class ChangeManager:
37
37
  for field in sender._meta.fields:
38
38
  payload["fields"][field.name] = str(getattr(instance, field.name))
39
39
  routing_key = f"{sender._meta.app_label}.{sender._meta.model_name}.save"
40
- message: EventType = make_event(payload, event=routing_key)
40
+ message: Event = make_event(payload, key=routing_key)
41
41
  self.notify(routing_key, message)
42
42
 
43
- def notify(self, routing_key: str, event: "EventType") -> bool:
43
+ def notify(self, routing_key: str, event: "Event") -> bool:
44
44
  logger.debug("notifying [%s] %s", routing_key, event)
45
45
  return self.backend.publish(routing_key, event)
46
46
 
@@ -74,7 +74,7 @@ class ThreadedChangeManager(ChangeManager):
74
74
  except queue.Empty:
75
75
  continue
76
76
 
77
- def notify(self, routing_key: str, event: "EventType") -> bool:
77
+ def notify(self, routing_key: str, event: "Event") -> bool:
78
78
  self.queue.put((routing_key, event))
79
79
  self.start()
80
80
  return True
@@ -1,6 +1,5 @@
1
1
  from collections.abc import Callable
2
- from datetime import datetime
3
- from typing import Literal, TypedDict
2
+ from typing import Literal
4
3
  from uuid import UUID
5
4
 
6
5
  from pika.adapters.blocking_connection import BlockingChannel
@@ -9,11 +8,6 @@ from pika.spec import Basic, BasicProperties
9
8
  JsonT = None | bool | int | float | str | list["JsonT"] | dict[str, "JsonT"] | UUID
10
9
  JSON = dict[str, JsonT]
11
10
 
11
+ EventType = Literal["absolute", "delta", "event"]
12
12
  PikaCallback = Callable[[BlockingChannel, Basic.Deliver, BasicProperties, bytes], None]
13
- UserCallback = Callable[[str, BlockingChannel, Basic.Deliver, BasicProperties, bytes], None]
14
-
15
- class EventType(TypedDict):
16
- timestamp: str | datetime
17
- event: str
18
- payload: JSON
19
- type: Literal["absolute", "delta", "event"]
13
+ UserCallback = Callable[[str, BlockingChannel, Basic.Deliver, BasicProperties, bytes], bool]
@@ -0,0 +1,111 @@
1
+ import logging
2
+ import socket
3
+ from collections.abc import Callable
4
+ from inspect import signature
5
+ from typing import TYPE_CHECKING, Any, get_type_hints
6
+
7
+ from colorama import Fore
8
+ from django.core.serializers.json import DjangoJSONEncoder
9
+ from django.db import models
10
+ from pika.adapters.blocking_connection import BlockingChannel
11
+ from pika.exceptions import ChannelClosedByBroker
12
+
13
+ from streaming.types import UserCallback
14
+
15
+ if TYPE_CHECKING:
16
+ from streaming.event import Event
17
+ from streaming.types import JSON
18
+
19
+ MINUTE = 60
20
+ HOUR = MINUTE * 60
21
+ DAY = HOUR * 24
22
+
23
+
24
+ class LevelFormatter(logging.Formatter):
25
+ template = f"$color$%(levelname)s{Fore.RESET}: {Fore.LIGHTWHITE_EX}%(name)s{Fore.RESET} - %(message)s"
26
+ default = Fore.LIGHTWHITE_EX
27
+ colors = {
28
+ logging.DEBUG: Fore.MAGENTA,
29
+ logging.CRITICAL: Fore.RED,
30
+ logging.ERROR: Fore.RED,
31
+ logging.INFO: Fore.BLUE,
32
+ logging.WARN: Fore.YELLOW,
33
+ }
34
+
35
+ def format(self, record: logging.LogRecord) -> str:
36
+ color = self.colors.get(record.levelno, self.default)
37
+ log_fmt = self.template.replace("$color$", color)
38
+ formatter = logging.Formatter(log_fmt)
39
+ return formatter.format(record)
40
+
41
+
42
+ class StreamingJSONEncoder(DjangoJSONEncoder):
43
+ def default(self, o: Any) -> Any:
44
+ if isinstance(o, models.Model):
45
+ return str(o)
46
+ return super().default(o)
47
+
48
+
49
+ def parse_bool(value: Any) -> bool:
50
+ if isinstance(value, str):
51
+ return value.lower() in ("yes", "true", "1", "y", "t")
52
+ return value in [1, True]
53
+
54
+
55
+ def make_event(message: "str | JSON", *, key: str = "") -> "Event":
56
+ from streaming.event import Event
57
+
58
+ return Event.build(key=key, data=message, value_type="absolute")
59
+
60
+
61
+ def get_local_ip() -> str:
62
+ s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
63
+ s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
64
+ s.connect(("<broadcast>", 12345)) # 12345 is random port. 0 fails on Mac.
65
+ return str(s.getsockname()[0])
66
+
67
+
68
+ def exchange_exists(channel: BlockingChannel | None, exchange_name: str) -> bool:
69
+ try:
70
+ channel.exchange_declare(exchange=exchange_name, passive=True) # type: ignore[union-attr]
71
+ return True
72
+ except (ChannelClosedByBroker, AttributeError):
73
+ return False
74
+
75
+
76
+ def check_callback(
77
+ func: Callable[
78
+ [
79
+ Any,
80
+ ],
81
+ Any,
82
+ ],
83
+ ) -> bool:
84
+ expected = UserCallback
85
+ try:
86
+ sig = signature(func)
87
+ params = list(sig.parameters.values())
88
+ hints = get_type_hints(func)
89
+
90
+ # For collections.abc.Callable, __args__ is ([arg1, arg2, ...], return_type)
91
+ expected_arg_types = expected.__args__[:-1] # type: ignore[attr-defined]
92
+ expected_return_type = expected.__args__[-1] # type: ignore[attr-defined]
93
+
94
+ if len(params) != len(expected_arg_types):
95
+ return False
96
+ return_hint = hints.get("return", type(None))
97
+ if expected_return_type is None:
98
+ expected_return_type = type(None)
99
+
100
+ for i, param in enumerate(params):
101
+ if hints.get(param.name) != expected_arg_types[i]:
102
+ return False
103
+
104
+ return bool(return_hint == expected_return_type)
105
+
106
+ except (TypeError, NameError, AttributeError, IndexError):
107
+ # TypeError -> not a callable or invalid annotations
108
+ # NameError -> forward refs not resolvable
109
+ # AttributeError -> in case expected is malformed
110
+ # IndexError -> if __args__ is not what we expect
111
+ return False
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
28
28
  commit_id: COMMIT_ID
29
29
  __commit_id__: COMMIT_ID
30
30
 
31
- __version__ = version = '0.5'
32
- __version_tuple__ = version_tuple = (0, 5)
31
+ __version__ = version = '0.7'
32
+ __version_tuple__ = version_tuple = (0, 7)
33
33
 
34
34
  __commit_id__ = commit_id = None
File without changes
@@ -1,27 +0,0 @@
1
- import logging
2
-
3
- logger = logging.getLogger(__name__)
4
-
5
-
6
- class StreamingError(Exception):
7
- pass
8
-
9
-
10
- class StreamingConfigError(StreamingError):
11
- pass
12
-
13
-
14
- class AuthorizationError(StreamingConfigError):
15
- pass
16
-
17
-
18
- class StreamingBackendError(StreamingError):
19
- pass
20
-
21
-
22
- class StreamingCallbackError(StreamingError):
23
- """Generic exception raised when a callback fails. Message will be re-queued."""
24
-
25
-
26
- class StreamingCallbackFailure(StreamingError): # noqa: N818
27
- """Generic exception raised when a callback fails. Message will be discarded."""
@@ -1,78 +0,0 @@
1
- import datetime
2
- import json
3
- import socket
4
- from typing import TYPE_CHECKING, Any
5
-
6
- from django.core.serializers.json import DjangoJSONEncoder
7
- from django.db import models
8
- from pika.adapters.blocking_connection import BlockingChannel
9
- from pika.exceptions import ChannelClosedByBroker
10
-
11
- if TYPE_CHECKING:
12
- from streaming.types import JSON, EventType
13
-
14
- MINUTE = 60
15
- HOUR = MINUTE * 60
16
- DAY = HOUR * 24
17
-
18
-
19
- class StreamingJSONEncoder(DjangoJSONEncoder):
20
- def default(self, o: Any) -> Any:
21
- if isinstance(o, datetime.datetime):
22
- return o.isoformat()
23
- if isinstance(o, models.Model):
24
- return str(o)
25
- return super().default(o)
26
-
27
-
28
- def json_dumps(obj: Any) -> str:
29
- return json.dumps(obj, cls=StreamingJSONEncoder)
30
-
31
-
32
- def json_loads(obj: str) -> "JSON":
33
- def try_parse_iso(value: Any) -> Any:
34
- if isinstance(value, str):
35
- try:
36
- return datetime.datetime.fromisoformat(value)
37
- except ValueError:
38
- return value
39
- return value
40
-
41
- def hook(dct: "JSON") -> Any:
42
- return {k: try_parse_iso(v) for k, v in dct.items()}
43
-
44
- return json.loads(obj, object_hook=hook) # type: ignore[no-any-return]
45
-
46
-
47
- def parse_bool(value: Any) -> bool:
48
- if isinstance(value, str):
49
- return value.lower() in ("yes", "true", "1", "y", "t")
50
- return value in [1, True]
51
-
52
-
53
- def make_event(message: "str | JSON", *, event: str = "") -> "EventType":
54
- if isinstance(message, str):
55
- payload: JSON = {"message": message}
56
- else:
57
- payload = message
58
- return {
59
- "event": event,
60
- "type": "absolute",
61
- "timestamp": datetime.datetime.now(),
62
- "payload": payload,
63
- }
64
-
65
-
66
- def get_local_ip() -> str:
67
- s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
68
- s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
69
- s.connect(("<broadcast>", 12345)) # 12345 is random port. 0 fails on Mac.
70
- return str(s.getsockname()[0])
71
-
72
-
73
- def exchange_exists(channel: BlockingChannel | None, exchange_name: str) -> bool:
74
- try:
75
- channel.exchange_declare(exchange=exchange_name, passive=True) # type: ignore[union-attr]
76
- return True
77
- except (ChannelClosedByBroker, AttributeError):
78
- return False
File without changes
File without changes