django-streaming 0.7__tar.gz → 0.7.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. {django_streaming-0.7 → django_streaming-0.7.2}/PKG-INFO +1 -1
  2. {django_streaming-0.7 → django_streaming-0.7.2}/src/streaming/__cli__.py +1 -1
  3. {django_streaming-0.7 → django_streaming-0.7.2}/src/streaming/callbacks.py +2 -1
  4. {django_streaming-0.7 → django_streaming-0.7.2}/src/streaming/config.py +1 -1
  5. {django_streaming-0.7 → django_streaming-0.7.2}/src/streaming/event.py +4 -7
  6. {django_streaming-0.7 → django_streaming-0.7.2}/src/streaming/manager.py +29 -8
  7. {django_streaming-0.7 → django_streaming-0.7.2}/src/streaming/utils.py +7 -4
  8. {django_streaming-0.7 → django_streaming-0.7.2}/src/streaming/version.py +2 -2
  9. {django_streaming-0.7 → django_streaming-0.7.2}/.gitignore +0 -0
  10. {django_streaming-0.7 → django_streaming-0.7.2}/LICENSE +0 -0
  11. {django_streaming-0.7 → django_streaming-0.7.2}/README.md +0 -0
  12. {django_streaming-0.7 → django_streaming-0.7.2}/pyproject.toml +0 -0
  13. {django_streaming-0.7 → django_streaming-0.7.2}/src/streaming/__init__.py +0 -0
  14. {django_streaming-0.7 → django_streaming-0.7.2}/src/streaming/apps.py +0 -0
  15. {django_streaming-0.7 → django_streaming-0.7.2}/src/streaming/backends/__init__.py +0 -0
  16. {django_streaming-0.7 → django_streaming-0.7.2}/src/streaming/backends/_base.py +0 -0
  17. {django_streaming-0.7 → django_streaming-0.7.2}/src/streaming/backends/console.py +0 -0
  18. {django_streaming-0.7 → django_streaming-0.7.2}/src/streaming/backends/debug.py +4 -4
  19. {django_streaming-0.7 → django_streaming-0.7.2}/src/streaming/backends/rabbitmq.py +0 -0
  20. {django_streaming-0.7 → django_streaming-0.7.2}/src/streaming/exceptions.py +0 -0
  21. {django_streaming-0.7 → django_streaming-0.7.2}/src/streaming/py.typed +0 -0
  22. {django_streaming-0.7 → django_streaming-0.7.2}/src/streaming/types.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: django-streaming
3
- Version: 0.7
3
+ Version: 0.7.2
4
4
  Summary: A project for streaming data.
5
5
  Project-URL: Documentation, https://github.com/UNICEF/django-streaming
6
6
  Project-URL: Homepage, https://github.com/UNICEF/django-streaming
@@ -109,7 +109,7 @@ def send(routing_key: str, message: str, client_name: str, debug: bool) -> None:
109
109
  payload = {
110
110
  "message": message,
111
111
  }
112
- msg: Event = make_event(payload, key="Test")
112
+ msg: Event = make_event(message=payload)
113
113
  backend.publish(routing_key, msg)
114
114
  click.secho(f"Sent: {msg}")
115
115
  backend.disconnect()
@@ -15,8 +15,9 @@ def default_callback(
15
15
  ) -> bool:
16
16
  logger.debug("Invoking default callback")
17
17
  message: Event = Event.unmarshal(body)
18
+ routing_key = method.routing_key
18
19
  ack = f"{Fore.GREEN}ACK{Fore.RESET}"
19
20
  sys.stdout.write(
20
- f"{Fore.GREEN}{message.timestamp} [{queue_name}]{Fore.LIGHTWHITE_EX} [{message.key}] {message.id} {ack}\n"
21
+ f"{Fore.GREEN}{message.timestamp} [{queue_name}]{Fore.LIGHTWHITE_EX} [{routing_key}] {message.id} {ack}\n"
21
22
  )
22
23
  return True
@@ -17,7 +17,7 @@ class StreamingConfig:
17
17
  "RETRY_COUNT": 3,
18
18
  "RETRY_DELAY": 1,
19
19
  "MESSAGE_TTL": 60 * 60 * 24 * 2, # 2 days
20
- "MANAGER_CLASS": "streaming.manager.ChangeManager",
20
+ "MANAGER_CLASS": "streaming.manager.ThreadedChangeManager",
21
21
  "LISTEN_CALLBACK": "streaming.callbacks.default_callback",
22
22
  }
23
23
 
@@ -13,14 +13,12 @@ class Event:
13
13
  def __init__(
14
14
  self,
15
15
  *,
16
- key: str,
17
16
  payload: "JSON",
18
17
  value_type: "EventType" = "absolute",
19
18
  timestamp: datetime | None = None,
20
19
  message_id: str | None = None,
21
20
  ) -> None:
22
21
  self.timestamp = timestamp or datetime.now()
23
- self.key = key
24
22
  self.payload = payload
25
23
  self.value_type = value_type
26
24
  self.id = message_id or uuid.uuid4()
@@ -30,7 +28,6 @@ class Event:
30
28
  {
31
29
  "id": self.id,
32
30
  "timestamp": self.timestamp.isoformat(),
33
- "key": self.key,
34
31
  "payload": self.payload,
35
32
  },
36
33
  cls=StreamingJSONEncoder,
@@ -42,23 +39,23 @@ class Event:
42
39
  return cls(
43
40
  message_id=data["id"],
44
41
  payload=data["payload"],
45
- key=data["key"],
46
42
  timestamp=datetime.fromisoformat(data["timestamp"]),
47
43
  )
48
44
 
49
45
  @classmethod
50
- def build(cls, key: str, data: Any, value_type: "EventType") -> "Event":
46
+ def build(
47
+ cls, data: Any, value_type: "EventType" = "absolute", key: str = "N/A", message_id: str | None = None
48
+ ) -> "Event":
51
49
  if isinstance(data, str):
52
50
  payload: JSON = {"message": data}
53
51
  else:
54
52
  payload = data
55
- return cls(key=key, payload=payload, value_type=value_type)
53
+ return cls(payload=payload, value_type=value_type, message_id=message_id)
56
54
 
57
55
  def as_dict(self) -> dict[str, Any]:
58
56
  return {
59
57
  "id": self.id,
60
58
  "timestamp": self.timestamp.isoformat(),
61
- "key": self.key,
62
59
  "payload": self.payload,
63
60
  "value_type": self.value_type,
64
61
  }
@@ -4,12 +4,14 @@ import queue
4
4
  import threading
5
5
  from typing import TYPE_CHECKING, Any
6
6
 
7
+ from django.db import models
7
8
  from django.db.models import Model
8
9
  from django.db.models.signals import post_save
9
10
  from django.utils.module_loading import import_string
10
11
 
11
12
  from streaming.backends import get_backend
12
13
 
14
+ from .exceptions import StreamingConfigError
13
15
  from .utils import make_event
14
16
 
15
17
  if TYPE_CHECKING:
@@ -20,24 +22,40 @@ logger = logging.getLogger(__name__)
20
22
 
21
23
  not_provided = object()
22
24
 
25
+ EXCLUDED_FIELDS = [
26
+ models.BinaryField,
27
+ models.FileField,
28
+ models.ImageField,
29
+ ]
30
+
31
+
32
+ def get_serializable_fields(model: type[Model]) -> list[str]:
33
+ return [f.name for f in model._meta.fields if f.__class__ not in EXCLUDED_FIELDS]
34
+
23
35
 
24
36
  class ChangeManager:
25
37
  def __init__(self) -> None:
26
- self._registry: set[type[Model]] = set()
38
+ self._registry: dict[type[Model], list[str]] = {}
39
+ self._registrations: list[str] = []
27
40
  self.backend: BaseBackend = get_backend()
28
41
 
29
- def register(self, model: type[Model], receiver: Any = None) -> None:
42
+ def register(self, model: type[Model], fields: list[str] | None = None, receiver: Any = None) -> None:
30
43
  logger.debug("Registering %s", model)
31
- self._registry.add(model)
32
- post_save.connect(receiver or self._post_save_receiver, sender=model, weak=False)
44
+ dispatch_uid = f"{id(self)}_{model.__name__}"
45
+ if dispatch_uid in self._registrations:
46
+ post_save.disconnect(dispatch_uid=dispatch_uid, sender=model)
47
+ else:
48
+ self._registrations.append(dispatch_uid)
49
+ self._registry[model] = fields or get_serializable_fields(model)
50
+ post_save.connect(receiver or self._post_save_receiver, sender=model, weak=False, dispatch_uid=dispatch_uid)
33
51
 
34
52
  def _post_save_receiver(self, sender: type[Model], instance: Model, created: bool, **kwargs: Any) -> None:
35
53
  logger.debug("post_save event for %s", sender)
36
54
  payload = {"model": sender.__name__, "pk": instance.pk, "created": created, "fields": {}}
37
- for field in sender._meta.fields:
38
- payload["fields"][field.name] = str(getattr(instance, field.name))
55
+ for field_name in self._registry[sender]:
56
+ payload["fields"][field_name] = str(getattr(instance, field_name))
39
57
  routing_key = f"{sender._meta.app_label}.{sender._meta.model_name}.save"
40
- message: Event = make_event(payload, key=routing_key)
58
+ message: Event = make_event(payload)
41
59
  self.notify(routing_key, message)
42
60
 
43
61
  def notify(self, routing_key: str, event: "Event") -> bool:
@@ -83,7 +101,10 @@ class ThreadedChangeManager(ChangeManager):
83
101
  def get_manager() -> ChangeManager:
84
102
  from streaming.config import CONFIG
85
103
 
86
- return import_string(CONFIG.MANAGER_CLASS)() # type: ignore[no-any-return]
104
+ try:
105
+ return import_string(CONFIG.MANAGER_CLASS)() # type: ignore[no-any-return]
106
+ except (ImportError, AttributeError):
107
+ raise StreamingConfigError("Invalid manager class. Check your django-streaming configuration.") from None
87
108
 
88
109
 
89
110
  def initialize_engine(reset: bool = False) -> ChangeManager | ThreadedChangeManager:
@@ -10,11 +10,10 @@ from django.db import models
10
10
  from pika.adapters.blocking_connection import BlockingChannel
11
11
  from pika.exceptions import ChannelClosedByBroker
12
12
 
13
- from streaming.types import UserCallback
13
+ from streaming.types import JSON, EventType, UserCallback
14
14
 
15
15
  if TYPE_CHECKING:
16
16
  from streaming.event import Event
17
- from streaming.types import JSON
18
17
 
19
18
  MINUTE = 60
20
19
  HOUR = MINUTE * 60
@@ -52,10 +51,14 @@ def parse_bool(value: Any) -> bool:
52
51
  return value in [1, True]
53
52
 
54
53
 
55
- def make_event(message: "str | JSON", *, key: str = "") -> "Event":
54
+ def make_event(message: "str|JSON", *, value_type: "EventType" = "absolute", message_id: str | None = None) -> "Event":
56
55
  from streaming.event import Event
57
56
 
58
- return Event.build(key=key, data=message, value_type="absolute")
57
+ return Event.build(
58
+ data=message,
59
+ value_type=value_type,
60
+ message_id=message_id,
61
+ )
59
62
 
60
63
 
61
64
  def get_local_ip() -> str:
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
28
28
  commit_id: COMMIT_ID
29
29
  __commit_id__: COMMIT_ID
30
30
 
31
- __version__ = version = '0.7'
32
- __version_tuple__ = version_tuple = (0, 7)
31
+ __version__ = version = '0.7.2'
32
+ __version_tuple__ = version_tuple = (0, 7, 2)
33
33
 
34
34
  __commit_id__ = commit_id = None
File without changes
@@ -11,16 +11,16 @@ if TYPE_CHECKING:
11
11
 
12
12
 
13
13
  class DebugBackend(BaseBackend):
14
+ def __init__(self, url: str) -> None:
15
+ super().__init__(url)
16
+ self.messages: list[tuple[str, Event]] = []
17
+
14
18
  def configure_queue_routing(self) -> dict[str, list[str]]: # pragma: no cover
15
19
  return {}
16
20
 
17
21
  def configure_exchanges(self) -> None: # pragma: no cover
18
22
  pass
19
23
 
20
- def __init__(self, url: str) -> None:
21
- super().__init__(url)
22
- self.messages: list[tuple[str, Event]] = []
23
-
24
24
  def listen(self, callback: "UserCallback", queues: list[str] | None = None, ack: bool = True) -> None:
25
25
  pass
26
26