django-streaming 0.5__tar.gz → 0.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. {django_streaming-0.5 → django_streaming-0.6.0}/PKG-INFO +1 -1
  2. {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/__cli__.py +47 -43
  3. {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/backends/_base.py +9 -6
  4. {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/backends/console.py +9 -2
  5. {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/backends/debug.py +10 -3
  6. {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/backends/rabbitmq.py +37 -30
  7. django_streaming-0.6.0/src/streaming/event.py +64 -0
  8. {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/exceptions.py +1 -1
  9. {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/manager.py +4 -4
  10. {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/types.pyi +2 -8
  11. {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/utils.py +7 -35
  12. {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/version.py +2 -2
  13. django_streaming-0.5/src/streaming/api.py +0 -0
  14. {django_streaming-0.5 → django_streaming-0.6.0}/.gitignore +0 -0
  15. {django_streaming-0.5 → django_streaming-0.6.0}/LICENSE +0 -0
  16. {django_streaming-0.5 → django_streaming-0.6.0}/README.md +0 -0
  17. {django_streaming-0.5 → django_streaming-0.6.0}/pyproject.toml +0 -0
  18. {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/__init__.py +0 -0
  19. {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/apps.py +0 -0
  20. {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/backends/__init__.py +0 -0
  21. {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/config.py +0 -0
  22. {django_streaming-0.5 → django_streaming-0.6.0}/src/streaming/py.typed +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: django-streaming
3
- Version: 0.5
3
+ Version: 0.6.0
4
4
  Summary: A project for streaming data.
5
5
  Project-URL: Documentation, https://github.com/UNICEF/django-streaming
6
6
  Project-URL: Homepage, https://github.com/UNICEF/django-streaming
@@ -11,28 +11,23 @@ from pika.exceptions import ChannelClosedByBroker
11
11
  from pika.spec import Basic, BasicProperties
12
12
 
13
13
  from .backends import RabbitMQBackend, get_backend
14
+ from .event import Event
14
15
  from .exceptions import AuthorizationError, StreamingConfigError
15
16
  from .utils import make_event
16
17
 
17
18
  if TYPE_CHECKING:
18
- from .types import JSON, EventType
19
-
19
+ from .types import JSON
20
20
 
21
21
  logger = logging.getLogger(__name__)
22
+ LINE = f"{Fore.YELLOW}%-16s: {Style.RESET_ALL}%s"
22
23
 
23
24
 
24
25
  def _dump_info(backend: "RabbitMQBackend") -> None:
25
- from streaming.config import CONFIG
26
-
27
- line = f"{Fore.YELLOW}%-16s: {Style.RESET_ALL}%s"
28
- click.secho(line % ("Server", f"{backend.host}:{backend.port}"))
29
- click.secho(line % ("VirtualHost", f"{backend.virtual_host}"))
30
- click.secho(line % ("Exchange", f"{backend.exchange}"))
31
- click.secho(line % ("Queues", ""))
32
- for alias, config in CONFIG.QUEUES.items():
33
- click.secho(line % (f" {alias}", f"{config}"))
34
- click.secho(line % ("Timeout", f"{backend.timeout}"))
35
- click.secho(line % ("Client Name", f"{backend.client_name}"))
26
+ click.secho(LINE % ("Server", f"{backend.host}:{backend.port}"))
27
+ click.secho(LINE % ("VirtualHost", f"{backend.virtual_host}"))
28
+ click.secho(LINE % ("Exchange", f"{backend.exchange}"))
29
+ click.secho(LINE % ("Timeout", f"{backend.timeout}"))
30
+ click.secho(LINE % ("Client Name", f"{backend.client_name}"))
36
31
 
37
32
 
38
33
  def assert_backend() -> "RabbitMQBackend":
@@ -43,6 +38,16 @@ def assert_backend() -> "RabbitMQBackend":
43
38
  return backend
44
39
 
45
40
 
41
+ def configure_logging(debug: bool, loggers: tuple[str, ...] = ("streaming",)) -> None:
42
+ for log_name in loggers:
43
+ logr = logging.getLogger(log_name)
44
+ if debug:
45
+ logr.setLevel(logging.DEBUG)
46
+ logr.addHandler(logging.StreamHandler())
47
+ else:
48
+ logr.handlers = []
49
+
50
+
46
51
  @click.group()
47
52
  def cli() -> None:
48
53
  """Streaming CLI."""
@@ -57,18 +62,24 @@ def cli() -> None:
57
62
 
58
63
 
59
64
  @cli.command()
60
- @click.option("--client-name", default=None, help="Override client name")
61
- def configure(client_name: str) -> None:
65
+ @click.option("--queues/--no-queues", "queues", is_flag=True, default=False, help="Debug mode")
66
+ @click.option("--debug", is_flag=True, help="Debug mode")
67
+ def configure(queues: bool = False, debug: bool = False) -> None:
62
68
  from streaming.config import CONFIG
63
69
 
70
+ routing = None
71
+ configure_logging(debug)
64
72
  backend: RabbitMQBackend = assert_backend()
65
- if client_name:
66
- backend.client_name = client_name
67
73
  try:
68
74
  backend.connect(True)
69
75
  backend.configure_exchanges()
70
- backend.configure_queue_routing()
76
+ if queues:
77
+ routing = backend.configure_queue_routing()
71
78
  check.callback() # type: ignore[misc]
79
+ if routing:
80
+ click.secho(LINE % ("Queues", ""))
81
+ for k, v in routing.items():
82
+ click.secho(LINE % (f" {k}", "; ".join(v)))
72
83
  except AuthorizationError as e:
73
84
  click.secho(f"Unable to connect using {CONFIG.BROKER_URL}", fg="red", err=True)
74
85
  raise ClickException(str(e)) from e
@@ -84,12 +95,7 @@ def configure(client_name: str) -> None:
84
95
  @click.option("--debug", is_flag=True, help="Debug mode")
85
96
  def send(routing_key: str, message: str, client_name: str, debug: bool) -> None:
86
97
  backend = assert_backend()
87
- logger = logging.getLogger("streaming")
88
- if debug:
89
- logger.setLevel(logging.DEBUG)
90
- logger.addHandler(logging.StreamHandler())
91
- else:
92
- logger.handlers = []
98
+ configure_logging(debug)
93
99
 
94
100
  if client_name:
95
101
  backend.client_name = client_name
@@ -99,40 +105,39 @@ def send(routing_key: str, message: str, client_name: str, debug: bool) -> None:
99
105
  payload = {
100
106
  "message": message,
101
107
  }
102
- msg: EventType = make_event(payload, event="Test")
108
+ msg: Event = make_event(payload, key="Test")
103
109
  backend.publish(routing_key, msg)
104
110
  click.secho(f"Sent: {msg}")
105
111
  backend.disconnect()
106
112
 
107
113
 
108
- def _listen(queues: list[str], payload: bool, pretty: bool, client_name: str) -> None:
114
+ def _listen(queues: list[str], payload: bool, pretty: bool) -> None:
109
115
  backend = assert_backend()
110
116
 
111
- if client_name:
112
- backend.client_name = client_name
113
-
114
117
  def callback(
115
118
  queue_name: str, ch: BlockingChannel, method: Basic.Deliver, properties: BasicProperties, body: bytes
116
119
  ) -> None:
117
- message: EventType = json.loads(body.decode())
118
- click.echo(
119
- f"{Fore.GREEN}{message['timestamp']} "
120
- f"[{queue_name}]"
121
- f"{Fore.LIGHTWHITE_EX} [{message['type']}]"
122
- f"{message['event']} "
123
- )
120
+ message: Event = Event.unmarshal(body)
121
+ click.echo(f"{Fore.GREEN}{message.timestamp} [{queue_name}]{Fore.LIGHTWHITE_EX} [{message.key}] {message.id} ")
124
122
  extra: str | JSON
125
123
  if payload:
126
124
  if pretty:
127
- extra = json.dumps(message["payload"], indent=4)
125
+ extra = json.dumps(message.payload, indent=4)
128
126
  else:
129
- extra = message["payload"]
127
+ extra = message.payload
130
128
  click.echo(f"{Fore.YELLOW}{extra}{Fore.RESET}")
131
129
 
132
130
  try:
133
- backend.connect()
131
+ backend.connect(True)
134
132
  _dump_info(backend)
135
133
  backend.listen(callback, queues=queues)
134
+ except ChannelClosedByBroker as e:
135
+ click.secho(str(e), fg="red")
136
+ if "no queue" in str(e):
137
+ click.secho("Did you run 'stream configure --queues'", fg="red")
138
+ except StreamingConfigError as e:
139
+ click.secho(str(e), fg="red")
140
+ click.get_current_context().exit(2)
136
141
  except KeyboardInterrupt:
137
142
  click.secho("Stopping listener.", fg="yellow")
138
143
  finally:
@@ -141,19 +146,18 @@ def _listen(queues: list[str], payload: bool, pretty: bool, client_name: str) ->
141
146
 
142
147
  @cli.command()
143
148
  @click.option("-q", "--queues", multiple=True, help="Queue name to listen to")
144
- @click.option("-c", "--client-name", default=None, help="Override client name")
145
149
  @click.option("--payload", default=False, is_flag=True, help="Print payload")
146
150
  @click.option("--autoreload", "reload", is_flag=True, help="Enable auto-reloading.")
147
151
  @click.option("--pretty", is_flag=True, help="Pretty-print payload.")
148
- def listen(queues: list[str], payload: bool, reload: bool, pretty: bool, client_name: str) -> None:
152
+ def listen(queues: list[str], payload: bool, reload: bool, pretty: bool) -> None:
149
153
  """Listens for streaming events."""
150
154
  if reload:
151
155
  from django.utils import autoreload
152
156
 
153
157
  click.secho("Starting listener with autoreload...", fg="yellow")
154
- autoreload.run_with_reloader(_listen, queues=queues, payload=payload, pretty=pretty, client_name=client_name)
158
+ autoreload.run_with_reloader(_listen, queues=queues, payload=payload, pretty=pretty)
155
159
  else:
156
- _listen(queues=queues, payload=payload, pretty=pretty, client_name=client_name)
160
+ _listen(queues=queues, payload=payload, pretty=pretty)
157
161
 
158
162
 
159
163
  @cli.command()
@@ -14,7 +14,8 @@ from ..exceptions import (
14
14
  if TYPE_CHECKING:
15
15
  from pika.adapters.blocking_connection import BlockingChannel, BlockingConnection
16
16
 
17
- from streaming.types import EventType, UserCallback
17
+ from ..event import Event
18
+ from ..types import UserCallback
18
19
 
19
20
 
20
21
  logger = logging.getLogger(__name__)
@@ -41,7 +42,7 @@ class BaseBackend(ABC):
41
42
  self.timeout = float(self.get_option("timeout", 0.5))
42
43
  self.virtual_host = self.get_option("vhost", "/")
43
44
  # listener
44
- self.client_name = CONFIG.CLIENT_NAME or get_local_ip()
45
+ self.client_name = CONFIG.CLIENT_NAME or self.get_option("client_name", get_local_ip())
45
46
  atexit.register(self.disconnect)
46
47
 
47
48
  def get_option(self, name: str, default: Any = "") -> Any:
@@ -55,11 +56,13 @@ class BaseBackend(ABC):
55
56
  def client_name(self, name: str) -> None:
56
57
  self.__client_name = name
57
58
 
59
+ @abstractmethod
58
60
  def configure_exchanges(self) -> None: # noqa: B027
59
- pass
61
+ ...
60
62
 
61
- def configure_queue_routing(self) -> None: # noqa: B027
62
- pass
63
+ @abstractmethod
64
+ def configure_queue_routing(self) -> dict[str, list[str]]: # noqa: B027
65
+ ...
63
66
 
64
67
  def get_real_queue_name(self, name: str) -> str:
65
68
  return f"{self.client_name}:{name}"
@@ -80,7 +83,7 @@ class BaseBackend(ABC):
80
83
 
81
84
  # Publisher
82
85
  @abstractmethod
83
- def publish(self, routing_key: str, message: "EventType") -> bool: ...
86
+ def publish(self, routing_key: str, message: "Event") -> bool: ...
84
87
 
85
88
  @abstractmethod
86
89
  def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
@@ -7,7 +7,8 @@ from ._base import BaseBackend
7
7
  logger = logging.getLogger(__name__)
8
8
 
9
9
  if TYPE_CHECKING:
10
- from ..types import EventType, UserCallback
10
+ from ..event import Event
11
+ from ..types import UserCallback
11
12
 
12
13
 
13
14
  class ConsoleBackend(BaseBackend):
@@ -15,7 +16,13 @@ class ConsoleBackend(BaseBackend):
15
16
  super().__init__(url)
16
17
  self.stream = self.get_option("stream", "stdout")
17
18
 
18
- def publish(self, routing_key: str, message: "EventType") -> bool:
19
+ def configure_queue_routing(self) -> dict[str, list[str]]: # pragma: no cover
20
+ return {}
21
+
22
+ def configure_exchanges(self) -> None: # pragma: no cover
23
+ pass
24
+
25
+ def publish(self, routing_key: str, message: "Event") -> bool:
19
26
  stream = getattr(sys, self.stream)
20
27
  stream.write(f"routing_key:{routing_key} message:{message}\n")
21
28
  return True
@@ -6,13 +6,20 @@ from ._base import BaseBackend
6
6
  logger = logging.getLogger(__name__)
7
7
 
8
8
  if TYPE_CHECKING:
9
- from ..types import EventType, UserCallback
9
+ from ..event import Event
10
+ from ..types import UserCallback
10
11
 
11
12
 
12
13
  class DebugBackend(BaseBackend):
14
+ def configure_queue_routing(self) -> dict[str, list[str]]: # pragma: no cover
15
+ return {}
16
+
17
+ def configure_exchanges(self) -> None: # pragma: no cover
18
+ pass
19
+
13
20
  def __init__(self, url: str) -> None:
14
21
  super().__init__(url)
15
- self.messages: list[tuple[str, EventType]] = []
22
+ self.messages: list[tuple[str, Event]] = []
16
23
 
17
24
  def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
18
25
  pass
@@ -23,6 +30,6 @@ class DebugBackend(BaseBackend):
23
30
  def disconnect(self) -> None:
24
31
  pass
25
32
 
26
- def publish(self, routing_key: str, message: "EventType") -> bool:
33
+ def publish(self, routing_key: str, message: "Event") -> bool:
27
34
  self.messages.append((routing_key, message))
28
35
  return True
@@ -1,7 +1,8 @@
1
- import json
2
1
  import logging
3
2
  import socket
4
3
  import time
4
+ from collections import defaultdict
5
+ from contextlib import suppress
5
6
  from typing import TYPE_CHECKING
6
7
 
7
8
  import pika.channel
@@ -13,20 +14,20 @@ from pika.exchange_type import ExchangeType
13
14
  from streaming.config import CONFIG
14
15
  from streaming.utils import exchange_exists
15
16
 
17
+ from ..event import Event
16
18
  from ..exceptions import (
17
19
  AuthorizationError,
18
- StreamingCallbackError,
19
20
  StreamingCallbackFailure,
21
+ StreamingCallbackRetryError,
20
22
  StreamingConfigError,
21
23
  )
22
- from ..utils import json_dumps
23
24
  from ._base import BaseBackend
24
25
 
25
26
  if TYPE_CHECKING:
26
27
  from pika.adapters.blocking_connection import BlockingChannel
27
28
  from pika.spec import Basic, BasicProperties
28
29
 
29
- from streaming.types import EventType, UserCallback
30
+ from streaming.types import UserCallback
30
31
 
31
32
  logger = logging.getLogger(__name__)
32
33
 
@@ -49,12 +50,11 @@ class Callback:
49
50
  self.user_callback(self.queue_name, ch, method, properties, body)
50
51
  if self.ack:
51
52
  ch.basic_ack(delivery_tag=method.delivery_tag) # type: ignore[arg-type]
52
- except StreamingCallbackError as e:
53
- evt: EventType = json.loads(body.decode())
53
+ except StreamingCallbackRetryError as e:
54
+ evt: Event = Event.unmarshal(body)
55
+ logger.debug(f"StreamingCallbackError {evt.id}", exc_info=e)
54
56
  retries = int(properties.headers.get("x-retries", 0)) # type: ignore[union-attr]
55
- ch.basic_ack(method.delivery_tag) # type: ignore[arg-type]
56
57
  self.backend._handle_retry(evt, ch, method, retries)
57
- logger.debug("StreamingCallbackError", exc_info=e)
58
58
  except StreamingCallbackFailure as e:
59
59
  logger.error(f"Callback failure: {e}", exc_info=e)
60
60
  except Exception as e:
@@ -89,19 +89,23 @@ class RabbitMQBackend(BaseBackend):
89
89
  )
90
90
  self.channel.exchange_declare(self.retry_exchange, exchange_type=ExchangeType.direct, durable=True)
91
91
 
92
- def configure_queue_routing(self) -> None:
92
+ def configure_queue_routing(self) -> dict[str, list[str]]:
93
93
  self.check_channel()
94
94
  if not exchange_exists(self.channel, self.exchange):
95
95
  raise StreamingConfigError("Exchange not found")
96
-
96
+ ret = defaultdict(list)
97
97
  for alias, config in CONFIG.QUEUES.items():
98
98
  real_name = self.get_real_queue_name(alias)
99
99
  logger.debug(f"Declaring queue '{real_name}'")
100
100
  self.channel.queue_declare(queue=real_name, durable=True, arguments=None) # type: ignore[union-attr]
101
- binding_keys = config.get("routing", [])
102
- for binding_key in binding_keys:
103
- logger.debug("Listening on queue '%s' routed by '%s'", alias, binding_key)
104
- self.channel.queue_bind(exchange=self.exchange, queue=real_name, routing_key=binding_key) # type: ignore[union-attr]
101
+ if binding_keys := config.get("routing", []):
102
+ for binding_key in binding_keys:
103
+ logger.debug("Listening on queue '%s' routed by '%s'", alias, binding_key)
104
+ self.channel.queue_bind(exchange=self.exchange, queue=real_name, routing_key=binding_key) # type: ignore[union-attr]
105
+ ret[alias].append(binding_key)
106
+ else:
107
+ ret[alias].append("warning no routing defined")
108
+ return ret
105
109
 
106
110
  def connect(self, raise_if_error: bool = False) -> None:
107
111
  logger.debug("Connecting to %s:%s", self.host, self.port)
@@ -148,16 +152,15 @@ class RabbitMQBackend(BaseBackend):
148
152
  def disconnect(self) -> None:
149
153
  try:
150
154
  if self._connection:
151
- logger.debug("Closing RabbitMQ connection.")
152
- self._connection.close()
153
- except (ConnectionClosedByBroker, AttributeError, ConnectionWrongStateError):
154
- pass
155
+ with suppress(ConnectionClosedByBroker, AttributeError, ConnectionWrongStateError):
156
+ logger.debug("Closing RabbitMQ connection.")
157
+ self._connection.close()
155
158
  finally:
156
159
  self._connection = None
157
160
  self.channel = None
158
161
 
159
162
  # Publisher
160
- def publish(self, routing_key: str, message: "EventType") -> bool:
163
+ def publish(self, routing_key: str, message: "Event") -> bool:
161
164
  try:
162
165
  if not self.channel or self.channel.is_closed:
163
166
  self.connect(True)
@@ -170,12 +173,14 @@ class RabbitMQBackend(BaseBackend):
170
173
 
171
174
  # Listener
172
175
 
173
- def _basic_publish(self, message: "EventType", routing_key: str, retry_count: int = 0) -> None:
176
+ def _basic_publish(
177
+ self, message: "Event", routing_key: str, retry_count: int = 0, exchange: str | None = None
178
+ ) -> None:
174
179
  self.check_channel()
175
180
  self.channel.basic_publish( # type: ignore[union-attr]
176
- exchange=self.exchange,
181
+ exchange=exchange or self.exchange,
177
182
  routing_key=routing_key,
178
- body=json_dumps(message).encode(),
183
+ body=message.marshall(),
179
184
  properties=pika.BasicProperties(
180
185
  delivery_mode=pika.spec.PERSISTENT_DELIVERY_MODE,
181
186
  expiration=str(CONFIG.MESSAGE_TTL * 1000), # milliseconds
@@ -183,11 +188,15 @@ class RabbitMQBackend(BaseBackend):
183
188
  ),
184
189
  )
185
190
 
186
- def _handle_retry(self, message: "EventType", ch: "BlockingChannel", method: "Basic.Deliver", retries: int) -> None:
191
+ def _handle_retry(self, message: "Event", ch: "BlockingChannel", method: "Basic.Deliver", retries: int) -> None:
187
192
  ch.basic_ack(method.delivery_tag) # type: ignore[arg-type]
188
193
  if retries < MAX_RETRIES:
189
194
  delay = 2000 * (2**retries) # ms (exponential backoff)
190
- delay_queue = f"{self.exchange}_retry_{delay}ms"
195
+ delay_queue = f"{self.exchange}_retry_{method.routing_key}_{delay}ms"
196
+ logger.debug(
197
+ f"Retrying message {message.id} ({retries + 1}/{MAX_RETRIES}) "
198
+ f"with routing key '{method.routing_key}' in {delay / 1000}s"
199
+ )
191
200
 
192
201
  # Declare a delay queue with TTL
193
202
  ch.queue_declare(
@@ -196,12 +205,13 @@ class RabbitMQBackend(BaseBackend):
196
205
  arguments={
197
206
  "x-dead-letter-exchange": self.exchange, # after delay → back to main
198
207
  "x-message-ttl": delay,
208
+ "x-dead-letter-routing-key": method.routing_key,
199
209
  },
200
210
  )
201
- ch.queue_bind(delay_queue, self.retry_exchange, routing_key="task")
202
- self._basic_publish(message, routing_key=method.routing_key or "*", retry_count=retries + 1)
211
+ ch.queue_bind(delay_queue, self.retry_exchange, routing_key=delay_queue)
212
+ self._basic_publish(message, routing_key=delay_queue, retry_count=retries + 1, exchange=self.retry_exchange)
203
213
  else:
204
- logger.error(f"Dropping after {MAX_RETRIES} retries")
214
+ logger.error(f"Dropping message {message.id} after {MAX_RETRIES} retries")
205
215
 
206
216
  def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
207
217
  if self.channel is None:
@@ -214,9 +224,6 @@ class RabbitMQBackend(BaseBackend):
214
224
  return
215
225
  queues_to_listen = queues or configured_queues.keys()
216
226
  for queue_name in queues_to_listen:
217
- if queue_name not in configured_queues:
218
- logger.error(f"Queue '{queue_name}' not found in configured listening queues. Ignored.")
219
- continue
220
227
  real_queue_name = self.get_real_queue_name(queue_name)
221
228
  _callback = Callback(queue_name, self, callback, ack=ack)
222
229
  self.channel.basic_consume(queue=real_queue_name, on_message_callback=_callback, auto_ack=False) # type: ignore[union-attr]
@@ -0,0 +1,64 @@
1
+ import json
2
+ import uuid
3
+ from datetime import datetime
4
+ from typing import TYPE_CHECKING, Any
5
+
6
+ from .utils import StreamingJSONEncoder
7
+
8
+ if TYPE_CHECKING:
9
+ from .types import JSON, EventType
10
+
11
+
12
+ class Event:
13
+ def __init__(
14
+ self,
15
+ *,
16
+ key: str,
17
+ payload: "JSON",
18
+ value_type: "EventType" = "absolute",
19
+ timestamp: datetime | None = None,
20
+ message_id: str | None = None,
21
+ ) -> None:
22
+ self.timestamp = timestamp or datetime.now()
23
+ self.key = key
24
+ self.payload = payload
25
+ self.value_type = value_type
26
+ self.id = message_id or uuid.uuid4()
27
+
28
+ def marshall(self) -> bytes:
29
+ return json.dumps(
30
+ {
31
+ "id": self.id,
32
+ "timestamp": self.timestamp.isoformat(),
33
+ "key": self.key,
34
+ "payload": self.payload,
35
+ },
36
+ cls=StreamingJSONEncoder,
37
+ ).encode()
38
+
39
+ @classmethod
40
+ def unmarshal(cls, body: bytes) -> "Event":
41
+ data = json.loads(body.decode())
42
+ return cls(
43
+ message_id=data["id"],
44
+ payload=data["payload"],
45
+ key=data["key"],
46
+ timestamp=datetime.fromisoformat(data["timestamp"]),
47
+ )
48
+
49
+ @classmethod
50
+ def build(cls, key: str, data: Any, value_type: "EventType") -> "Event":
51
+ if isinstance(data, str):
52
+ payload: JSON = {"message": data}
53
+ else:
54
+ payload = data
55
+ return cls(key=key, payload=payload, value_type=value_type)
56
+
57
+ def as_dict(self) -> dict[str, Any]:
58
+ return {
59
+ "id": self.id,
60
+ "timestamp": self.timestamp.isoformat(),
61
+ "key": self.key,
62
+ "payload": self.payload,
63
+ "value_type": self.value_type,
64
+ }
@@ -19,7 +19,7 @@ class StreamingBackendError(StreamingError):
19
19
  pass
20
20
 
21
21
 
22
- class StreamingCallbackError(StreamingError):
22
+ class StreamingCallbackRetryError(StreamingError):
23
23
  """Generic exception raised when a callback fails. Message will be re-queued."""
24
24
 
25
25
 
@@ -14,7 +14,7 @@ from .utils import make_event
14
14
 
15
15
  if TYPE_CHECKING:
16
16
  from .backends._base import BaseBackend
17
- from .types import EventType
17
+ from .event import Event
18
18
 
19
19
  logger = logging.getLogger(__name__)
20
20
 
@@ -37,10 +37,10 @@ class ChangeManager:
37
37
  for field in sender._meta.fields:
38
38
  payload["fields"][field.name] = str(getattr(instance, field.name))
39
39
  routing_key = f"{sender._meta.app_label}.{sender._meta.model_name}.save"
40
- message: EventType = make_event(payload, event=routing_key)
40
+ message: Event = make_event(payload, key=routing_key)
41
41
  self.notify(routing_key, message)
42
42
 
43
- def notify(self, routing_key: str, event: "EventType") -> bool:
43
+ def notify(self, routing_key: str, event: "Event") -> bool:
44
44
  logger.debug("notifying [%s] %s", routing_key, event)
45
45
  return self.backend.publish(routing_key, event)
46
46
 
@@ -74,7 +74,7 @@ class ThreadedChangeManager(ChangeManager):
74
74
  except queue.Empty:
75
75
  continue
76
76
 
77
- def notify(self, routing_key: str, event: "EventType") -> bool:
77
+ def notify(self, routing_key: str, event: "Event") -> bool:
78
78
  self.queue.put((routing_key, event))
79
79
  self.start()
80
80
  return True
@@ -1,6 +1,5 @@
1
1
  from collections.abc import Callable
2
- from datetime import datetime
3
- from typing import Literal, TypedDict
2
+ from typing import Literal
4
3
  from uuid import UUID
5
4
 
6
5
  from pika.adapters.blocking_connection import BlockingChannel
@@ -9,11 +8,6 @@ from pika.spec import Basic, BasicProperties
9
8
  JsonT = None | bool | int | float | str | list["JsonT"] | dict[str, "JsonT"] | UUID
10
9
  JSON = dict[str, JsonT]
11
10
 
11
+ EventType = Literal["absolute", "delta", "event"]
12
12
  PikaCallback = Callable[[BlockingChannel, Basic.Deliver, BasicProperties, bytes], None]
13
13
  UserCallback = Callable[[str, BlockingChannel, Basic.Deliver, BasicProperties, bytes], None]
14
-
15
- class EventType(TypedDict):
16
- timestamp: str | datetime
17
- event: str
18
- payload: JSON
19
- type: Literal["absolute", "delta", "event"]
@@ -1,5 +1,3 @@
1
- import datetime
2
- import json
3
1
  import socket
4
2
  from typing import TYPE_CHECKING, Any
5
3
 
@@ -9,7 +7,9 @@ from pika.adapters.blocking_connection import BlockingChannel
9
7
  from pika.exceptions import ChannelClosedByBroker
10
8
 
11
9
  if TYPE_CHECKING:
12
- from streaming.types import JSON, EventType
10
+ from streaming.event import Event
11
+ from streaming.types import JSON
12
+
13
13
 
14
14
  MINUTE = 60
15
15
  HOUR = MINUTE * 60
@@ -18,49 +18,21 @@ DAY = HOUR * 24
18
18
 
19
19
  class StreamingJSONEncoder(DjangoJSONEncoder):
20
20
  def default(self, o: Any) -> Any:
21
- if isinstance(o, datetime.datetime):
22
- return o.isoformat()
23
21
  if isinstance(o, models.Model):
24
22
  return str(o)
25
23
  return super().default(o)
26
24
 
27
25
 
28
- def json_dumps(obj: Any) -> str:
29
- return json.dumps(obj, cls=StreamingJSONEncoder)
30
-
31
-
32
- def json_loads(obj: str) -> "JSON":
33
- def try_parse_iso(value: Any) -> Any:
34
- if isinstance(value, str):
35
- try:
36
- return datetime.datetime.fromisoformat(value)
37
- except ValueError:
38
- return value
39
- return value
40
-
41
- def hook(dct: "JSON") -> Any:
42
- return {k: try_parse_iso(v) for k, v in dct.items()}
43
-
44
- return json.loads(obj, object_hook=hook) # type: ignore[no-any-return]
45
-
46
-
47
26
  def parse_bool(value: Any) -> bool:
48
27
  if isinstance(value, str):
49
28
  return value.lower() in ("yes", "true", "1", "y", "t")
50
29
  return value in [1, True]
51
30
 
52
31
 
53
- def make_event(message: "str | JSON", *, event: str = "") -> "EventType":
54
- if isinstance(message, str):
55
- payload: JSON = {"message": message}
56
- else:
57
- payload = message
58
- return {
59
- "event": event,
60
- "type": "absolute",
61
- "timestamp": datetime.datetime.now(),
62
- "payload": payload,
63
- }
32
+ def make_event(message: "str | JSON", *, key: str = "") -> "Event":
33
+ from streaming.event import Event
34
+
35
+ return Event.build(key=key, data=message, value_type="absolute")
64
36
 
65
37
 
66
38
  def get_local_ip() -> str:
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
28
28
  commit_id: COMMIT_ID
29
29
  __commit_id__: COMMIT_ID
30
30
 
31
- __version__ = version = '0.5'
32
- __version_tuple__ = version_tuple = (0, 5)
31
+ __version__ = version = '0.6.0'
32
+ __version_tuple__ = version_tuple = (0, 6, 0)
33
33
 
34
34
  __commit_id__ = commit_id = None
File without changes
File without changes