django-streaming 0.7.1__tar.gz → 0.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. {django_streaming-0.7.1 → django_streaming-0.8.0}/PKG-INFO +1 -1
  2. {django_streaming-0.7.1 → django_streaming-0.8.0}/src/streaming/__cli__.py +1 -1
  3. {django_streaming-0.7.1 → django_streaming-0.8.0}/src/streaming/callbacks.py +2 -1
  4. {django_streaming-0.7.1 → django_streaming-0.8.0}/src/streaming/config.py +1 -1
  5. {django_streaming-0.7.1 → django_streaming-0.8.0}/src/streaming/event.py +1 -6
  6. {django_streaming-0.7.1 → django_streaming-0.8.0}/src/streaming/manager.py +29 -8
  7. {django_streaming-0.7.1 → django_streaming-0.8.0}/src/streaming/utils.py +1 -4
  8. {django_streaming-0.7.1 → django_streaming-0.8.0}/src/streaming/version.py +2 -2
  9. {django_streaming-0.7.1 → django_streaming-0.8.0}/.gitignore +0 -0
  10. {django_streaming-0.7.1 → django_streaming-0.8.0}/LICENSE +0 -0
  11. {django_streaming-0.7.1 → django_streaming-0.8.0}/README.md +0 -0
  12. {django_streaming-0.7.1 → django_streaming-0.8.0}/pyproject.toml +0 -0
  13. {django_streaming-0.7.1 → django_streaming-0.8.0}/src/streaming/__init__.py +0 -0
  14. {django_streaming-0.7.1 → django_streaming-0.8.0}/src/streaming/apps.py +0 -0
  15. {django_streaming-0.7.1 → django_streaming-0.8.0}/src/streaming/backends/__init__.py +0 -0
  16. {django_streaming-0.7.1 → django_streaming-0.8.0}/src/streaming/backends/_base.py +0 -0
  17. {django_streaming-0.7.1 → django_streaming-0.8.0}/src/streaming/backends/console.py +0 -0
  18. {django_streaming-0.7.1 → django_streaming-0.8.0}/src/streaming/backends/debug.py +4 -4
  19. {django_streaming-0.7.1 → django_streaming-0.8.0}/src/streaming/backends/rabbitmq.py +0 -0
  20. {django_streaming-0.7.1 → django_streaming-0.8.0}/src/streaming/exceptions.py +0 -0
  21. {django_streaming-0.7.1 → django_streaming-0.8.0}/src/streaming/py.typed +0 -0
  22. {django_streaming-0.7.1 → django_streaming-0.8.0}/src/streaming/types.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: django-streaming
3
- Version: 0.7.1
3
+ Version: 0.8.0
4
4
  Summary: A project for streaming data.
5
5
  Project-URL: Documentation, https://github.com/UNICEF/django-streaming
6
6
  Project-URL: Homepage, https://github.com/UNICEF/django-streaming
@@ -109,7 +109,7 @@ def send(routing_key: str, message: str, client_name: str, debug: bool) -> None:
109
109
  payload = {
110
110
  "message": message,
111
111
  }
112
- msg: Event = make_event(message=payload, key="Test")
112
+ msg: Event = make_event(message=payload)
113
113
  backend.publish(routing_key, msg)
114
114
  click.secho(f"Sent: {msg}")
115
115
  backend.disconnect()
@@ -15,8 +15,9 @@ def default_callback(
15
15
  ) -> bool:
16
16
  logger.debug("Invoking default callback")
17
17
  message: Event = Event.unmarshal(body)
18
+ routing_key = method.routing_key
18
19
  ack = f"{Fore.GREEN}ACK{Fore.RESET}"
19
20
  sys.stdout.write(
20
- f"{Fore.GREEN}{message.timestamp} [{queue_name}]{Fore.LIGHTWHITE_EX} [{message.key}] {message.id} {ack}\n"
21
+ f"{Fore.GREEN}{message.timestamp} [{queue_name}]{Fore.LIGHTWHITE_EX} [{routing_key}] {message.id} {ack}\n"
21
22
  )
22
23
  return True
@@ -17,7 +17,7 @@ class StreamingConfig:
17
17
  "RETRY_COUNT": 3,
18
18
  "RETRY_DELAY": 1,
19
19
  "MESSAGE_TTL": 60 * 60 * 24 * 2, # 2 days
20
- "MANAGER_CLASS": "streaming.manager.ChangeManager",
20
+ "MANAGER_CLASS": "streaming.manager.ThreadedChangeManager",
21
21
  "LISTEN_CALLBACK": "streaming.callbacks.default_callback",
22
22
  }
23
23
 
@@ -13,14 +13,12 @@ class Event:
13
13
  def __init__(
14
14
  self,
15
15
  *,
16
- key: str,
17
16
  payload: "JSON",
18
17
  value_type: "EventType" = "absolute",
19
18
  timestamp: datetime | None = None,
20
19
  message_id: str | None = None,
21
20
  ) -> None:
22
21
  self.timestamp = timestamp or datetime.now()
23
- self.key = key
24
22
  self.payload = payload
25
23
  self.value_type = value_type
26
24
  self.id = message_id or uuid.uuid4()
@@ -30,7 +28,6 @@ class Event:
30
28
  {
31
29
  "id": self.id,
32
30
  "timestamp": self.timestamp.isoformat(),
33
- "key": self.key,
34
31
  "payload": self.payload,
35
32
  },
36
33
  cls=StreamingJSONEncoder,
@@ -42,7 +39,6 @@ class Event:
42
39
  return cls(
43
40
  message_id=data["id"],
44
41
  payload=data["payload"],
45
- key=data["key"],
46
42
  timestamp=datetime.fromisoformat(data["timestamp"]),
47
43
  )
48
44
 
@@ -54,13 +50,12 @@ class Event:
54
50
  payload: JSON = {"message": data}
55
51
  else:
56
52
  payload = data
57
- return cls(key=key, payload=payload, value_type=value_type, message_id=message_id)
53
+ return cls(payload=payload, value_type=value_type, message_id=message_id)
58
54
 
59
55
  def as_dict(self) -> dict[str, Any]:
60
56
  return {
61
57
  "id": self.id,
62
58
  "timestamp": self.timestamp.isoformat(),
63
- "key": self.key,
64
59
  "payload": self.payload,
65
60
  "value_type": self.value_type,
66
61
  }
@@ -4,12 +4,14 @@ import queue
4
4
  import threading
5
5
  from typing import TYPE_CHECKING, Any
6
6
 
7
+ from django.db import models
7
8
  from django.db.models import Model
8
9
  from django.db.models.signals import post_save
9
10
  from django.utils.module_loading import import_string
10
11
 
11
12
  from streaming.backends import get_backend
12
13
 
14
+ from .exceptions import StreamingConfigError
13
15
  from .utils import make_event
14
16
 
15
17
  if TYPE_CHECKING:
@@ -20,24 +22,40 @@ logger = logging.getLogger(__name__)
20
22
 
21
23
  not_provided = object()
22
24
 
25
+ EXCLUDED_FIELDS = [
26
+ models.BinaryField,
27
+ models.FileField,
28
+ models.ImageField,
29
+ ]
30
+
31
+
32
+ def get_serializable_fields(model: type[Model]) -> list[str]:
33
+ return [f.name for f in model._meta.fields if f.__class__ not in EXCLUDED_FIELDS]
34
+
23
35
 
24
36
  class ChangeManager:
25
37
  def __init__(self) -> None:
26
- self._registry: set[type[Model]] = set()
38
+ self._registry: dict[type[Model], list[str]] = {}
39
+ self._registrations: list[str] = []
27
40
  self.backend: BaseBackend = get_backend()
28
41
 
29
- def register(self, model: type[Model], receiver: Any = None) -> None:
42
+ def register(self, model: type[Model], fields: list[str] | None = None, receiver: Any = None) -> None:
30
43
  logger.debug("Registering %s", model)
31
- self._registry.add(model)
32
- post_save.connect(receiver or self._post_save_receiver, sender=model, weak=False)
44
+ dispatch_uid = f"{id(self)}_{model.__name__}"
45
+ if dispatch_uid in self._registrations:
46
+ post_save.disconnect(dispatch_uid=dispatch_uid, sender=model)
47
+ else:
48
+ self._registrations.append(dispatch_uid)
49
+ self._registry[model] = fields or get_serializable_fields(model)
50
+ post_save.connect(receiver or self._post_save_receiver, sender=model, weak=False, dispatch_uid=dispatch_uid)
33
51
 
34
52
  def _post_save_receiver(self, sender: type[Model], instance: Model, created: bool, **kwargs: Any) -> None:
35
53
  logger.debug("post_save event for %s", sender)
36
54
  payload = {"model": sender.__name__, "pk": instance.pk, "created": created, "fields": {}}
37
- for field in sender._meta.fields:
38
- payload["fields"][field.name] = str(getattr(instance, field.name))
55
+ for field_name in self._registry[sender]:
56
+ payload["fields"][field_name] = str(getattr(instance, field_name))
39
57
  routing_key = f"{sender._meta.app_label}.{sender._meta.model_name}.save"
40
- message: Event = make_event(payload, key=routing_key)
58
+ message: Event = make_event(payload)
41
59
  self.notify(routing_key, message)
42
60
 
43
61
  def notify(self, routing_key: str, event: "Event") -> bool:
@@ -83,7 +101,10 @@ class ThreadedChangeManager(ChangeManager):
83
101
  def get_manager() -> ChangeManager:
84
102
  from streaming.config import CONFIG
85
103
 
86
- return import_string(CONFIG.MANAGER_CLASS)() # type: ignore[no-any-return]
104
+ try:
105
+ return import_string(CONFIG.MANAGER_CLASS)() # type: ignore[no-any-return]
106
+ except (ImportError, AttributeError):
107
+ raise StreamingConfigError("Invalid manager class. Check your django-streaming configuration.") from None
87
108
 
88
109
 
89
110
  def initialize_engine(reset: bool = False) -> ChangeManager | ThreadedChangeManager:
@@ -51,13 +51,10 @@ def parse_bool(value: Any) -> bool:
51
51
  return value in [1, True]
52
52
 
53
53
 
54
- def make_event(
55
- message: "str|JSON", *, value_type: "EventType" = "absolute", key: str = "N/A", message_id: str | None = None
56
- ) -> "Event":
54
+ def make_event(message: "str|JSON", *, value_type: "EventType" = "absolute", message_id: str | None = None) -> "Event":
57
55
  from streaming.event import Event
58
56
 
59
57
  return Event.build(
60
- key=key,
61
58
  data=message,
62
59
  value_type=value_type,
63
60
  message_id=message_id,
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
28
28
  commit_id: COMMIT_ID
29
29
  __commit_id__: COMMIT_ID
30
30
 
31
- __version__ = version = '0.7.1'
32
- __version_tuple__ = version_tuple = (0, 7, 1)
31
+ __version__ = version = '0.8.0'
32
+ __version_tuple__ = version_tuple = (0, 8, 0)
33
33
 
34
34
  __commit_id__ = commit_id = None
@@ -11,16 +11,16 @@ if TYPE_CHECKING:
11
11
 
12
12
 
13
13
  class DebugBackend(BaseBackend):
14
+ def __init__(self, url: str) -> None:
15
+ super().__init__(url)
16
+ self.messages: list[tuple[str, Event]] = []
17
+
14
18
  def configure_queue_routing(self) -> dict[str, list[str]]: # pragma: no cover
15
19
  return {}
16
20
 
17
21
  def configure_exchanges(self) -> None: # pragma: no cover
18
22
  pass
19
23
 
20
- def __init__(self, url: str) -> None:
21
- super().__init__(url)
22
- self.messages: list[tuple[str, Event]] = []
23
-
24
24
  def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
25
25
  pass
26
26